From 17794d72686178154e38a0b9e49ac74160b4200c Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 5 Aug 2026 12:27:45 -0700 Subject: [PATCH 01/26] [PERF]: Avoid deep copies with GIL in pylibcudf read_parquet_footers This changes the ownership model of FileMetadata to own the footers via unique_ptr and move them (under nogil) in the pylibcudf wrapper. Additionally, we avoid an unnecessary copy in get_parquet_metadatas(). --- cpp/src/io/parquet/reader_impl_helpers.hpp | 23 ++++++- .../pylibcudf/io/experimental/hybrid_scan.pyx | 11 +++- python/pylibcudf/pylibcudf/io/parquet.pyx | 2 +- .../pylibcudf/io/parquet_metadata.pxd | 8 ++- .../pylibcudf/io/parquet_metadata.pyx | 65 ++++++++++++++----- 5 files changed, 83 insertions(+), 26 deletions(-) diff --git a/cpp/src/io/parquet/reader_impl_helpers.hpp b/cpp/src/io/parquet/reader_impl_helpers.hpp index ed67ca05cfc3..59cfe5e65ee6 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.hpp +++ b/cpp/src/io/parquet/reader_impl_helpers.hpp @@ -469,15 +469,34 @@ class aggregate_reader_metadata { std::span input_columns) const; /** - * @brief Get Parquet file metadatas + * @brief Get Parquet file metadatas (copies) * * @return Parquet file metadatas */ - [[nodiscard]] std::vector get_parquet_metadatas() const + [[nodiscard]] std::vector get_parquet_metadatas() const& { return std::vector{per_file_metadata.begin(), per_file_metadata.end()}; } + /** + * @brief Get Parquet file metadatas by moving out of this aggregate + * + * Used when the aggregate_reader_metadata is temporary and discarded after + * extracting footers (e.g. read_parquet_footers), avoiding a deep copy. + * + * @return Parquet file metadatas + */ + [[nodiscard]] std::vector get_parquet_metadatas() && + { + std::vector result; + result.reserve(per_file_metadata.size()); + for (auto& pfm : per_file_metadata) { + result.push_back(std::move(static_cast(pfm))); + } + per_file_metadata.clear(); + return result; + } + /** * @brief Extracts the schema_idx'th column chunk metadata from row_group_index'th row group of * the src_idx'th file. diff --git a/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx b/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx index 664eb489428c..89b951dd0ecd 100644 --- a/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx +++ b/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from cython.operator cimport dereference from libc.stdint cimport uint8_t, uintptr_t from libc.stddef cimport size_t from libcpp.memory cimport make_unique, unique_ptr @@ -26,6 +27,7 @@ from pylibcudf.libcudf.io.hybrid_scan cimport ( hybrid_scan_reader as cpp_hybrid_scan_reader, use_data_page_mask as cpp_use_data_page_mask, ) +from pylibcudf.libcudf.io.parquet_schema cimport FileMetaData as cpp_FileMetaData from pylibcudf.libcudf.io.text cimport byte_range_info from pylibcudf.libcudf.io.types cimport table_with_metadata from pylibcudf.libcudf.types cimport size_type @@ -104,7 +106,7 @@ cdef class HybridScanReader: """ cdef HybridScanReader reader = HybridScanReader.__new__(HybridScanReader) reader.c_obj = make_unique[cpp_hybrid_scan_reader]( - metadata.c_obj, + dereference(metadata.c_obj), options.c_obj ) return reader @@ -117,7 +119,12 @@ cdef class HybridScanReader: FileMetaData Parquet file footer metadata """ - return c_FileMetaData.from_cpp(self.c_obj.get()[0].parquet_metadata()) + cdef unique_ptr[cpp_FileMetaData] metadata + with nogil: + metadata = make_unique[cpp_FileMetaData]( + self.c_obj.get()[0].parquet_metadata() + ) + return c_FileMetaData.from_libcudf(move(metadata)) def page_index_byte_range(self): """Get the byte range of the page index. diff --git a/python/pylibcudf/pylibcudf/io/parquet.pyx b/python/pylibcudf/pylibcudf/io/parquet.pyx index d287d10900ec..81410fadafc7 100644 --- a/python/pylibcudf/pylibcudf/io/parquet.pyx +++ b/python/pylibcudf/pylibcudf/io/parquet.pyx @@ -93,7 +93,7 @@ cdef vector[cpp_FileMetaData] _build_parquet_metadatas( raise TypeError( "parquet_metadatas must contain only FileMetaData objects" ) - metadata_ptrs.push_back(&(metadata).c_obj) + metadata_ptrs.push_back((metadata).c_obj.get()) if metadata_ptrs.size() != num_sources: raise ValueError( diff --git a/python/pylibcudf/pylibcudf/io/parquet_metadata.pxd b/python/pylibcudf/pylibcudf/io/parquet_metadata.pxd index 67fdda5d6907..fedbca1800f2 100644 --- a/python/pylibcudf/pylibcudf/io/parquet_metadata.pxd +++ b/python/pylibcudf/pylibcudf/io/parquet_metadata.pxd @@ -1,6 +1,8 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from libcpp.memory cimport unique_ptr + from pylibcudf.io.types cimport SourceInfo from pylibcudf.libcudf.io.parquet_schema cimport ( ColumnChunk as cpp_ColumnChunk, @@ -65,10 +67,10 @@ cdef class ParquetMetadata: cpdef dict columnchunk_metadata(self) cdef class FileMetaData: - cdef cpp_FileMetaData c_obj + cdef unique_ptr[cpp_FileMetaData] c_obj @staticmethod - cdef FileMetaData from_cpp(cpp_FileMetaData metadata) + cdef FileMetaData from_libcudf(unique_ptr[cpp_FileMetaData] metadata) cdef class SortingColumn: cdef cpp_SortingColumn c_obj diff --git a/python/pylibcudf/pylibcudf/io/parquet_metadata.pyx b/python/pylibcudf/pylibcudf/io/parquet_metadata.pyx index f8aca270642e..2042a6f2bd28 100644 --- a/python/pylibcudf/pylibcudf/io/parquet_metadata.pyx +++ b/python/pylibcudf/pylibcudf/io/parquet_metadata.pyx @@ -1,9 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from cython.operator cimport dereference from libc.stdint cimport uint8_t from libcpp.memory cimport make_unique, unique_ptr from libcpp.string cimport string +from libcpp.utility cimport move from libcpp.vector cimport vector from pylibcudf.io.types cimport SourceInfo @@ -467,31 +469,34 @@ cdef class FileMetaData: raise ValueError("FileMetaData cannot be constructed directly") @staticmethod - cdef FileMetaData from_cpp(cpp_FileMetaData metadata): + cdef FileMetaData from_libcudf(unique_ptr[cpp_FileMetaData] metadata): cdef FileMetaData result = FileMetaData.__new__(FileMetaData) - result.c_obj = metadata + result.c_obj = move(metadata) return result @property def version(self): """Get the file format version.""" - return self.c_obj.version + return dereference(self.c_obj).version @property def num_rows(self): """Get the total number of rows.""" - return self.c_obj.num_rows + return dereference(self.c_obj).num_rows @property def created_by(self): """Get the application that created the file.""" - return self.c_obj.created_by.decode("utf-8") + return dereference(self.c_obj).created_by.decode("utf-8") @property def row_groups(self): """Get row group metadata in this file.""" cdef cpp_RowGroup row_group - return [RowGroup.from_cpp(row_group) for row_group in self.c_obj.row_groups] + return [ + RowGroup.from_cpp(row_group) + for row_group in dereference(self.c_obj).row_groups + ] @property def row_group_num_rows(self): @@ -512,8 +517,10 @@ cdef class FileMetaData: >>> [rg.num_rows for rg in file_metadata.row_groups] """ cdef Py_ssize_t i - cdef Py_ssize_t n = self.c_obj.row_groups.size() - return [self.c_obj.row_groups[i].num_rows for i in range(n)] + cdef Py_ssize_t n = dereference(self.c_obj).row_groups.size() + return [ + dereference(self.c_obj).row_groups[i].num_rows for i in range(n) + ] @property def columnchunk_metadata(self): @@ -544,21 +551,33 @@ cdef class FileMetaData: ... ) """ cdef Py_ssize_t i, j, k, n_path, n_col - cdef Py_ssize_t n_rg = self.c_obj.row_groups.size() + cdef Py_ssize_t n_rg = dereference(self.c_obj).row_groups.size() cdef dict result = {} cdef str name cdef list path_parts for i in range(n_rg): - n_col = self.c_obj.row_groups[i].columns.size() + n_col = dereference(self.c_obj).row_groups[i].columns.size() for j in range(n_col): - n_path = self.c_obj.row_groups[i].columns[j].meta_data.path_in_schema.size() + n_path = ( + dereference(self.c_obj) + .row_groups[i] + .columns[j] + .meta_data.path_in_schema.size() + ) path_parts = [ - self.c_obj.row_groups[i].columns[j].meta_data.path_in_schema[k].decode("utf-8") + dereference(self.c_obj) + .row_groups[i] + .columns[j] + .meta_data.path_in_schema[k] + .decode("utf-8") for k in range(n_path) ] name = ".".join(path_parts) result.setdefault(name, []).append( - self.c_obj.row_groups[i].columns[j].meta_data.total_uncompressed_size + dereference(self.c_obj) + .row_groups[i] + .columns[j] + .meta_data.total_uncompressed_size ) return result @@ -583,7 +602,7 @@ cdef class FileMetaData: """ cdef parquet_reader_options options = parquet_reader_options() cdef unique_ptr[cpp_hybrid_scan_reader] reader - cdef cpp_FileMetaData metadata + cdef unique_ptr[cpp_FileMetaData] metadata cdef const uint8_t* footer_ptr = 0 if len(footer_bytes) > 0: @@ -594,9 +613,11 @@ cdef class FileMetaData: host_span[const_uint8_t](footer_ptr, len(footer_bytes)), options, ) - metadata = reader.get()[0].parquet_metadata() + metadata = make_unique[cpp_FileMetaData]( + reader.get()[0].parquet_metadata() + ) - return FileMetaData.from_cpp(metadata) + return FileMetaData.from_libcudf(move(metadata)) cpdef ParquetMetadata read_parquet_metadata(SourceInfo src_info): @@ -644,7 +665,8 @@ cpdef list read_parquet_footers(SourceInfo src_info): """ cdef vector[unique_ptr[datasource]] sources cdef vector[cpp_FileMetaData] c_result - cdef cpp_FileMetaData metadata + cdef vector[unique_ptr[cpp_FileMetaData]] owned + cdef size_t i, n with nogil: sources = make_datasources(src_info.c_obj) c_result = cpp_parquet_metadata.read_parquet_footers( @@ -653,5 +675,12 @@ cpdef list read_parquet_footers(SourceInfo src_info): sources.size(), ) ) + n = c_result.size() + owned.reserve(n) + for i in range(n): + owned.push_back( + move(make_unique[cpp_FileMetaData](move(c_result[i]))) + ) - return [FileMetaData.from_cpp(metadata) for metadata in c_result] + # GIL held only for Python object allocation + list build + return [FileMetaData.from_libcudf(move(owned[i])) for i in range(n)] From ecfd5b6cb00b48de79d2427e1500dc9a5e369a94 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 6 Aug 2026 05:39:12 -0700 Subject: [PATCH 02/26] Attempt to reduce GIL contention in Scan nodes. --- cpp/include/cudf/io/detail/parquet.hpp | 13 ++++++ cpp/src/io/parquet/reader_impl.cpp | 14 ++++++ python/pylibcudf/pylibcudf/io/parquet.pyx | 44 +++++++++++++------ .../pylibcudf/libcudf/io/parquet.pxd | 9 ++++ 4 files changed, 66 insertions(+), 14 deletions(-) diff --git a/cpp/include/cudf/io/detail/parquet.hpp b/cpp/include/cudf/io/detail/parquet.hpp index 3ec2090ad032..41fc140629d9 100644 --- a/cpp/include/cudf/io/detail/parquet.hpp +++ b/cpp/include/cudf/io/detail/parquet.hpp @@ -266,6 +266,19 @@ parquet_metadata read_parquet_metadata(host_span con std::vector read_parquet_footers( std::span const> sources); +/** + * @brief Deep-copy FileMetaData objects from a span of pointers + * + * Used when handing cached footers into the parquet reader so the clone can + * run without the Python GIL (Cython calls this under `nogil`). + * + * @param sources Non-owning pointers to FileMetaData objects to clone + * + * @return Deep copies of each pointed-to FileMetaData + */ +[[nodiscard]] std::vector clone_parquet_metadatas( + host_span sources); + } // namespace parquet::detail } // namespace io } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 307015ec2c3f..24115920ef94 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -1212,4 +1212,18 @@ std::vector read_parquet_footers( .get_parquet_metadatas(); } +[[nodiscard]] std::vector clone_parquet_metadatas( + host_span sources) +{ + CUDF_FUNC_RANGE(); + + std::vector result; + result.reserve(sources.size()); + for (auto const* src : sources) { + CUDF_EXPECTS(src != nullptr, "Null FileMetaData pointer"); + result.push_back(*src); + } + return result; +} + } // namespace cudf::io::parquet::detail diff --git a/python/pylibcudf/pylibcudf/io/parquet.pyx b/python/pylibcudf/pylibcudf/io/parquet.pyx index 81410fadafc7..8f5a70a7dbc4 100644 --- a/python/pylibcudf/pylibcudf/io/parquet.pyx +++ b/python/pylibcudf/pylibcudf/io/parquet.pyx @@ -28,6 +28,8 @@ from pylibcudf.libcudf.expressions cimport expression from pylibcudf.libcudf.io.datasource cimport datasource, make_datasources from pylibcudf.libcudf.io.parquet cimport ( chunked_parquet_reader as cpp_chunked_parquet_reader, + clone_parquet_metadatas, + const_FileMetaData_ptr, parquet_reader_options, read_parquet as cpp_read_parquet, write_parquet as cpp_write_parquet, @@ -38,6 +40,7 @@ from pylibcudf.libcudf.io.parquet cimport ( chunked_parquet_writer_options, merge_row_group_metadata as cpp_merge_row_group_metadata, ) +from pylibcudf.libcudf.utilities.span cimport host_span from pylibcudf.libcudf.io.parquet_schema cimport FileMetaData as cpp_FileMetaData from pylibcudf.libcudf.io.types cimport ( compression_type, @@ -77,16 +80,20 @@ def _warn_deprecated(api_name, new_api): ) -cdef vector[cpp_FileMetaData] _build_parquet_metadatas( +cdef vector[cpp_FileMetaData*] _parquet_metadata_ptrs( object parquet_metadatas, size_t num_sources, ) except *: - cdef vector[cpp_FileMetaData] c_metadatas + """Validate Python FileMetaData list and collect non-owning C++ pointers. + + The expensive deep clone must happen later under the same ``nogil`` block as + ``read_parquet`` / the chunked reader ctor. Returning ``vector[FileMetaData]`` + from a cdef helper copies again with the GIL held (no libcudf NVTX). + """ cdef vector[cpp_FileMetaData*] metadata_ptrs cdef object metadata - cdef size_t i if parquet_metadatas is None: - return c_metadatas + return metadata_ptrs for metadata in parquet_metadatas: if not isinstance(metadata, FileMetaData): @@ -102,14 +109,7 @@ cdef vector[cpp_FileMetaData] _build_parquet_metadatas( f"({num_sources})" ) - c_metadatas.reserve(metadata_ptrs.size()) - with nogil: - # This copies the (potentially large) metadata object. We don't - # want to hold the GIL for that. - for i in range(metadata_ptrs.size()): - c_metadatas.push_back(dereference(metadata_ptrs[i])) - - return c_metadatas + return metadata_ptrs cdef class ParquetReaderOptions: @@ -612,6 +612,7 @@ cdef class ChunkedParquetReader: self.mr = _get_memory_resource(mr) cdef vector[unique_ptr[datasource]] sources cdef vector[cpp_FileMetaData] c_metadatas + cdef vector[cpp_FileMetaData*] metadata_ptrs cdef cudaStream_t stream_view = self._stream.view().value() if parquet_metadatas is None: with nogil: @@ -627,10 +628,16 @@ cdef class ChunkedParquetReader: else: with nogil: sources = make_datasources(options.c_obj.get_source()) - c_metadatas = _build_parquet_metadatas( + metadata_ptrs = _parquet_metadata_ptrs( parquet_metadatas, sources.size() ) with nogil: + c_metadatas = clone_parquet_metadatas( + host_span[const_FileMetaData_ptr]( + metadata_ptrs.data(), + metadata_ptrs.size(), + ) + ) self.reader.reset( new cpp_chunked_parquet_reader( chunk_read_limit, @@ -711,16 +718,25 @@ cpdef read_parquet( cdef cudaStream_t _cs = s.view().value() cdef vector[unique_ptr[datasource]] sources cdef vector[cpp_FileMetaData] c_metadatas + cdef vector[cpp_FileMetaData*] metadata_ptrs cdef table_with_metadata c_result mr = _get_memory_resource(mr) if parquet_metadatas is None: with nogil: c_result = move(cpp_read_parquet(options.c_obj, _cs, mr.get_mr())) else: + # Collect pointers under GIL; clone + read must share one nogil block so + # Cython does not deep-copy vector[FileMetaData] while holding the GIL. with nogil: sources = make_datasources(options.c_obj.get_source()) - c_metadatas = _build_parquet_metadatas(parquet_metadatas, sources.size()) + metadata_ptrs = _parquet_metadata_ptrs(parquet_metadatas, sources.size()) with nogil: + c_metadatas = clone_parquet_metadatas( + host_span[const_FileMetaData_ptr]( + metadata_ptrs.data(), + metadata_ptrs.size(), + ) + ) c_result = move( cpp_read_parquet( move(sources), diff --git a/python/pylibcudf/pylibcudf/libcudf/io/parquet.pxd b/python/pylibcudf/pylibcudf/libcudf/io/parquet.pxd index 0f5a531dea8a..3a922e90790d 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/parquet.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/parquet.pxd @@ -351,3 +351,12 @@ cdef extern from "cudf/io/parquet.hpp" namespace "cudf::io" nogil: cdef unique_ptr[vector[uint8_t]] merge_row_group_metadata( const vector[unique_ptr[vector[uint8_t]]]& metadata_list ) except +libcudf_exception_handler + +from pylibcudf.libcudf.utilities.span cimport host_span + +ctypedef const FileMetaData* const_FileMetaData_ptr + +cdef extern from "cudf/io/detail/parquet.hpp" namespace "cudf::io::parquet::detail" nogil: + cdef vector[FileMetaData] clone_parquet_metadatas( + host_span[const_FileMetaData_ptr] sources + ) except +libcudf_exception_handler From b3775f55f91d691e0aefc99ef157d2a9c62c20e1 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 6 Aug 2026 10:39:40 -0700 Subject: [PATCH 03/26] Option to skip page index materialization in parquet metadata When page indexes are present, cloning the parquet footer FileMetaData object can be expensive. This slows down `read_parquet` when the user provides a prefetched metadata object. This PR adds a new `read_page_indexes` parameter to `read_parquet_footers` that controls whether page indexes are materialized. The default is `True` matching the existing behavior. --- .../io/parquet/parquet_reader_metadata.cpp | 4 +- cpp/include/cudf/io/detail/parquet.hpp | 3 +- cpp/include/cudf/io/parquet.hpp | 3 +- cpp/include/cudf/io/parquet_metadata.hpp | 10 +++- cpp/src/io/functions.cpp | 4 +- cpp/src/io/parquet/reader_impl.cpp | 8 ++- cpp/tests/io/parquet_reader_test.cpp | 52 +++++++++++++++++++ .../cudf_polars/cudf_polars/dsl/utils/io.py | 5 +- .../pylibcudf/io/parquet_metadata.pxd | 3 +- .../pylibcudf/io/parquet_metadata.pyi | 4 +- .../pylibcudf/io/parquet_metadata.pyx | 15 +++++- .../pylibcudf/libcudf/io/parquet_metadata.pxd | 6 ++- python/pylibcudf/tests/io/test_parquet.py | 4 +- 13 files changed, 101 insertions(+), 20 deletions(-) diff --git a/cpp/benchmarks/io/parquet/parquet_reader_metadata.cpp b/cpp/benchmarks/io/parquet/parquet_reader_metadata.cpp index 28f5ada2adf8..72fd2e9760d1 100644 --- a/cpp/benchmarks/io/parquet/parquet_reader_metadata.cpp +++ b/cpp/benchmarks/io/parquet/parquet_reader_metadata.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -125,7 +125,7 @@ void BM_parquet_read_footer(nvbench::state& state) auto sources = cudf::io::make_datasources(source_info); timer.start(); - auto const metadatas = cudf::io::read_parquet_footers(sources); + auto const metadatas = cudf::io::read_parquet_footers(sources, write_page_index); timer.stop(); // Validate metadata diff --git a/cpp/include/cudf/io/detail/parquet.hpp b/cpp/include/cudf/io/detail/parquet.hpp index 41fc140629d9..a9c4d7fda18d 100644 --- a/cpp/include/cudf/io/detail/parquet.hpp +++ b/cpp/include/cudf/io/detail/parquet.hpp @@ -260,11 +260,12 @@ parquet_metadata read_parquet_metadata(host_span con * @brief Constructs FileMetaData objects from parquet dataset * * @param sources Input `datasource` objects to read the dataset from + * @param read_page_indexes If true, deserialize page indexes into each column chunk when present * * @return List of FileMetaData objects, one per parquet source */ std::vector read_parquet_footers( - std::span const> sources); + std::span const> sources, bool read_page_indexes = true); /** * @brief Deep-copy FileMetaData objects from a span of pointers diff --git a/cpp/include/cudf/io/parquet.hpp b/cpp/include/cudf/io/parquet.hpp index 148dc15b5c69..2561f84326ab 100644 --- a/cpp/include/cudf/io/parquet.hpp +++ b/cpp/include/cudf/io/parquet.hpp @@ -978,7 +978,8 @@ table_with_metadata read_parquet( * The following code snippet demonstrates how to read a dataset from a file: * @code * auto sources = cudf::io::make_datasources(cudf::io::source_info("dataset.parquet")); - * auto metadatas = cudf::io::read_parquet_footers(sources); + * // Pass read_page_indexes=false when reusing only with read_parquet (page indexes optional). + * auto metadatas = cudf::io::read_parquet_footers(sources, false); * auto options = cudf::io::parquet_reader_options::builder(); * auto result = cudf::io::read_parquet(std::move(sources), std::move(metadatas), options); * @endcode diff --git a/cpp/include/cudf/io/parquet_metadata.hpp b/cpp/include/cudf/io/parquet_metadata.hpp index c6c0e6f4e376..915ff6112b96 100644 --- a/cpp/include/cudf/io/parquet_metadata.hpp +++ b/cpp/include/cudf/io/parquet_metadata.hpp @@ -285,14 +285,22 @@ parquet_metadata read_parquet_metadata(source_info const& src_info); /** * @brief Constructs FileMetaData objects from parquet dataset * + * By default (`read_page_indexes == true`), page indexes (`ColumnIndex` / `OffsetIndex`) are + * deserialized into each column chunk when present. Page indexes are not required by + * `read_parquet`; pass `false` when reusing footers only with `read_parquet` or + * `chunked_parquet_reader` to avoid the cost of materializing and later deep-cloning them. + * Hybrid-scan callers that need page-level stats can either keep the default or call + * `setup_page_index` separately. + * * @ingroup io_readers * * @param sources Input `datasource` objects to read the dataset from + * @param read_page_indexes If true, deserialize page indexes into each column chunk when present * * @return List of FileMetaData objects, one per parquet source */ std::vector read_parquet_footers( - std::span const> sources); + std::span const> sources, bool read_page_indexes = true); /** @} */ // end of group } // namespace io diff --git a/cpp/src/io/functions.cpp b/cpp/src/io/functions.cpp index 5322cb40fc99..2b0a691d5a1e 100644 --- a/cpp/src/io/functions.cpp +++ b/cpp/src/io/functions.cpp @@ -678,10 +678,10 @@ parquet_metadata read_parquet_metadata(source_info const& src_info) } std::vector read_parquet_footers( - std::span const> sources) + std::span const> sources, bool read_page_indexes) { CUDF_FUNC_RANGE(); - return detail_parquet::read_parquet_footers(sources); + return detail_parquet::read_parquet_footers(sources, read_page_indexes); } /** diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 24115920ef94..6d54927f4272 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -1192,7 +1192,7 @@ parquet_metadata read_parquet_metadata(host_span con } std::vector read_parquet_footers( - std::span const> sources) + std::span const> sources, bool read_page_indexes) { // Do not use arrow schema when only reading the parquet metadata. constexpr auto use_arrow_schema = false; @@ -1200,10 +1200,8 @@ std::vector read_parquet_footers( // Do not select any columns when only reading the parquet metadata. constexpr auto has_column_projection = false; - // Read page indexes if available here since we will want to reuse the raw metadata for later use. - constexpr auto read_page_indexes = true; - - // Parse the source dataset metadata + // Parse the source dataset metadata. Callers that only reuse footers with read_parquet can + // pass read_page_indexes=false to skip materializing ColumnIndex/OffsetIndex (cheaper to clone). return aggregate_reader_metadata( host_span const>{sources.data(), sources.size()}, use_arrow_schema, diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index 6ac067e74d3f..57046565df0d 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -3159,6 +3159,58 @@ TEST_F(ParquetMetadataReaderTest, PreMaterializedMetadata) test_parquet_metadata(3); } +TEST_F(ParquetMetadataReaderTest, ReadParquetFootersPageIndexes) +{ + // Page indexes are only deserialized when BYTE_ARRAY columns are present. + auto const num_rows = 2000; + std::vector strings(num_rows); + std::generate( + strings.begin(), strings.end(), [i = 0]() mutable { return "str_" + std::to_string(i++); }); + cudf::test::strings_column_wrapper str_col(strings.begin(), strings.end()); + table_view input_table({str_col}); + + auto filepath = temp_env->get_temp_filepath("ReadParquetFootersPageIndexes.parquet"); + cudf::io::parquet_writer_options out_opts = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, input_table) + .stats_level(cudf::io::statistics_freq::STATISTICS_COLUMN) + .row_group_size_rows(500) + .build(); + cudf::io::write_parquet(out_opts); + + auto const source_info = cudf::io::source_info{filepath}; + auto datasources = cudf::io::make_datasources(source_info); + + // Explicit false omits page indexes (lean path for read_parquet reuse). + { + auto metadatas = cudf::io::read_parquet_footers(datasources, false); + ASSERT_EQ(metadatas.size(), 1); + ASSERT_FALSE(metadatas.front().row_groups.empty()); + ASSERT_FALSE(metadatas.front().row_groups.front().columns.empty()); + EXPECT_FALSE(metadatas.front().row_groups.front().columns.front().offset_index.has_value()); + EXPECT_FALSE(metadatas.front().row_groups.front().columns.front().column_index.has_value()); + // File still records page-index byte ranges when written with column stats. + EXPECT_GT(metadatas.front().row_groups.front().columns.front().column_index_offset, 0); + EXPECT_GT(metadatas.front().row_groups.front().columns.front().column_index_length, 0); + + auto const options = cudf::io::parquet_reader_options::builder(source_info).build(); + auto sources_copy = cudf::io::make_datasources(source_info); + auto const read = + cudf::io::read_parquet(std::move(sources_copy), std::move(metadatas), options); + auto const expected = cudf::io::read_parquet(options); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected.tbl->view(), read.tbl->view()); + } + + // Default (true) materializes page indexes when present. + { + auto metadatas = cudf::io::read_parquet_footers(datasources); + ASSERT_EQ(metadatas.size(), 1); + ASSERT_FALSE(metadatas.front().row_groups.empty()); + ASSERT_FALSE(metadatas.front().row_groups.front().columns.empty()); + EXPECT_TRUE(metadatas.front().row_groups.front().columns.front().offset_index.has_value()); + EXPECT_TRUE(metadatas.front().row_groups.front().columns.front().column_index.has_value()); + } +} + TEST_F(ParquetMetadataReaderTest, Nested) { auto const num_rows = 1200; diff --git a/python/cudf_polars/cudf_polars/dsl/utils/io.py b/python/cudf_polars/cudf_polars/dsl/utils/io.py index 75373afeed96..9693165dc1ae 100644 --- a/python/cudf_polars/cudf_polars/dsl/utils/io.py +++ b/python/cudf_polars/cudf_polars/dsl/utils/io.py @@ -90,13 +90,16 @@ def _prefetch_parquet_footers_for_paths(paths: list[str]) -> list[CachedParquetI else: sizes.append(None) + # Page indexes are not required by read_parquet; omit them to keep prefetched + # metadata lean for reuse/clone. metadata = plc.io.parquet_metadata.read_parquet_footers( plc.io.types.SourceInfo( [ plc.io.types.FilepathSource(path, size) for path, size in zip(paths, sizes, strict=True) ] - ) + ), + read_page_indexes=False, ) return [ diff --git a/python/pylibcudf/pylibcudf/io/parquet_metadata.pxd b/python/pylibcudf/pylibcudf/io/parquet_metadata.pxd index fedbca1800f2..634b856e5d1a 100644 --- a/python/pylibcudf/pylibcudf/io/parquet_metadata.pxd +++ b/python/pylibcudf/pylibcudf/io/parquet_metadata.pxd @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from libcpp cimport bool from libcpp.memory cimport unique_ptr from pylibcudf.io.types cimport SourceInfo @@ -97,4 +98,4 @@ cdef class RowGroup: cdef RowGroup from_cpp(cpp_RowGroup row_group) cpdef ParquetMetadata read_parquet_metadata(SourceInfo src_info) -cpdef list read_parquet_footers(SourceInfo src_info) +cpdef list read_parquet_footers(SourceInfo src_info, bool read_page_indexes=*) diff --git a/python/pylibcudf/pylibcudf/io/parquet_metadata.pyi b/python/pylibcudf/pylibcudf/io/parquet_metadata.pyi index 6aa9efb19713..49f5d60f10fa 100644 --- a/python/pylibcudf/pylibcudf/io/parquet_metadata.pyi +++ b/python/pylibcudf/pylibcudf/io/parquet_metadata.pyi @@ -110,4 +110,6 @@ class RowGroup: def ordinal(self) -> int | None: ... def read_parquet_metadata(src_info: SourceInfo) -> ParquetMetadata: ... -def read_parquet_footers(src_info: SourceInfo) -> list[FileMetaData]: ... +def read_parquet_footers( + src_info: SourceInfo, read_page_indexes: bool = True +) -> list[FileMetaData]: ... diff --git a/python/pylibcudf/pylibcudf/io/parquet_metadata.pyx b/python/pylibcudf/pylibcudf/io/parquet_metadata.pyx index 2042a6f2bd28..0f0386961c29 100644 --- a/python/pylibcudf/pylibcudf/io/parquet_metadata.pyx +++ b/python/pylibcudf/pylibcudf/io/parquet_metadata.pyx @@ -3,6 +3,7 @@ from cython.operator cimport dereference from libc.stdint cimport uint8_t +from libcpp cimport bool from libcpp.memory cimport make_unique, unique_ptr from libcpp.string cimport string from libcpp.utility cimport move @@ -649,14 +650,22 @@ cpdef ParquetMetadata read_parquet_metadata(SourceInfo src_info): return ParquetMetadata.from_metadata(c_result) -cpdef list read_parquet_footers(SourceInfo src_info): +cpdef list read_parquet_footers(SourceInfo src_info, bool read_page_indexes=True): """ Read parquet file footers as ``FileMetaData`` objects. + By default, page indexes (``ColumnIndex`` / ``OffsetIndex``) are deserialized + when present. They are not required by + :func:`~pylibcudf.io.parquet.read_parquet`; pass ``read_page_indexes=False`` + when reusing footers only with ``read_parquet`` or ``ChunkedParquetReader`` + to avoid materializing (and later deep-cloning) them. + Parameters ---------- src_info : SourceInfo Dataset source. + read_page_indexes : bool, default True + If True, deserialize page indexes into each column chunk when present. Returns ------- @@ -667,13 +676,15 @@ cpdef list read_parquet_footers(SourceInfo src_info): cdef vector[cpp_FileMetaData] c_result cdef vector[unique_ptr[cpp_FileMetaData]] owned cdef size_t i, n + cdef bool c_read_page_indexes = read_page_indexes with nogil: sources = make_datasources(src_info.c_obj) c_result = cpp_parquet_metadata.read_parquet_footers( host_span[const_unique_ptr_datasource]( sources.data(), sources.size(), - ) + ), + c_read_page_indexes, ) n = c_result.size() owned.reserve(n) diff --git a/python/pylibcudf/pylibcudf/libcudf/io/parquet_metadata.pxd b/python/pylibcudf/pylibcudf/libcudf/io/parquet_metadata.pxd index 3a2b41de2774..f0b7aec61a26 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/parquet_metadata.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/parquet_metadata.pxd @@ -1,6 +1,7 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libc.stdint cimport int64_t +from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.string cimport string from libcpp.unordered_map cimport unordered_map @@ -45,5 +46,6 @@ cdef extern from "cudf/io/parquet_metadata.hpp" namespace "cudf::io" nogil: ) except +libcudf_exception_handler cdef vector[FileMetaData] read_parquet_footers( - host_span[const_unique_ptr_datasource] sources + host_span[const_unique_ptr_datasource] sources, + bool read_page_indexes ) except +libcudf_exception_handler diff --git a/python/pylibcudf/tests/io/test_parquet.py b/python/pylibcudf/tests/io/test_parquet.py index be56621c2bc9..5cdc2540ac5d 100644 --- a/python/pylibcudf/tests/io/test_parquet.py +++ b/python/pylibcudf/tests/io/test_parquet.py @@ -318,9 +318,11 @@ def test_read_parquet_from_device_buffers( assert_table_and_meta_eq(expected, res, check_field_nullability=False) +@pytest.mark.parametrize("read_page_indexes", [False, True]) def test_read_parquet_with_pre_materialized_metadata( table_data: tuple[plc.io.types.TableWithMetadata, pa.Table], binary_source_or_sink: str | os.PathLike[str] | io.BytesIO, + read_page_indexes: bool, ) -> None: _, pa_table = table_data source = make_source( @@ -330,7 +332,7 @@ def test_read_parquet_with_pre_materialized_metadata( options = plc.io.parquet.ParquetReaderOptions.builder(source_info).build() parquet_metadatas = plc.io.parquet_metadata.read_parquet_footers( - source_info + source_info, read_page_indexes=read_page_indexes ) result = plc.io.parquet.read_parquet( options, parquet_metadatas=parquet_metadatas From f5525b59b93ea8219c988a0de049fd4593d6ab0c Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 6 Aug 2026 12:37:41 -0700 Subject: [PATCH 04/26] Enable prefetching by default --- python/cudf_polars/cudf_polars/utils/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cudf_polars/cudf_polars/utils/config.py b/python/cudf_polars/cudf_polars/utils/config.py index 63fc734a0ca7..9c06469d5c58 100644 --- a/python/cudf_polars/cudf_polars/utils/config.py +++ b/python/cudf_polars/cudf_polars/utils/config.py @@ -225,7 +225,7 @@ class ParquetOptions: Default is False. prefetch_file_metadata Whether to prefetch parquet file metadata and pass it through - `parquet_metadatas` to avoid rereading file footers. + `parquet_metadatas` to avoid rereading file footers. Enabled by default. use_jit_filter Whether to use JIT compilation for post-read filtering in Parquet scans. When enabled, filter predicates are JIT-compiled to CUDA kernels for @@ -276,7 +276,7 @@ class ParquetOptions: default_factory=_make_default_factory( f"{_env_prefix}__PREFETCH_FILE_METADATA", _bool_converter, - default=False, + default=True, ) ) use_jit_filter: bool = dataclasses.field( From 0053fb4efe05fb3a39676b9d8d2bf0e531f03b90 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 6 Aug 2026 13:29:04 -0700 Subject: [PATCH 05/26] test style --- cpp/tests/io/parquet_reader_test.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index 57046565df0d..d443a2d24e49 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -3163,10 +3163,9 @@ TEST_F(ParquetMetadataReaderTest, ReadParquetFootersPageIndexes) { // Page indexes are only deserialized when BYTE_ARRAY columns are present. auto const num_rows = 2000; - std::vector strings(num_rows); - std::generate( - strings.begin(), strings.end(), [i = 0]() mutable { return "str_" + std::to_string(i++); }); - cudf::test::strings_column_wrapper str_col(strings.begin(), strings.end()); + auto str_iter = cudf::detail::make_counting_transform_iterator( + 0, [](auto i) { return "str_" + std::to_string(i); }); + cudf::test::strings_column_wrapper str_col(str_iter, str_iter + num_rows); table_view input_table({str_col}); auto filepath = temp_env->get_temp_filepath("ReadParquetFootersPageIndexes.parquet"); @@ -3183,7 +3182,7 @@ TEST_F(ParquetMetadataReaderTest, ReadParquetFootersPageIndexes) // Explicit false omits page indexes (lean path for read_parquet reuse). { auto metadatas = cudf::io::read_parquet_footers(datasources, false); - ASSERT_EQ(metadatas.size(), 1); + EXPECT_EQ(metadatas.size(), 1); ASSERT_FALSE(metadatas.front().row_groups.empty()); ASSERT_FALSE(metadatas.front().row_groups.front().columns.empty()); EXPECT_FALSE(metadatas.front().row_groups.front().columns.front().offset_index.has_value()); @@ -3203,7 +3202,7 @@ TEST_F(ParquetMetadataReaderTest, ReadParquetFootersPageIndexes) // Default (true) materializes page indexes when present. { auto metadatas = cudf::io::read_parquet_footers(datasources); - ASSERT_EQ(metadatas.size(), 1); + EXPECT_EQ(metadatas.size(), 1); ASSERT_FALSE(metadatas.front().row_groups.empty()); ASSERT_FALSE(metadatas.front().row_groups.front().columns.empty()); EXPECT_TRUE(metadatas.front().row_groups.front().columns.front().offset_index.has_value()); From 34914751c089c1911056663fc4e8d471a7e6e88c Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Fri, 7 Aug 2026 12:47:57 -0700 Subject: [PATCH 06/26] prefetch only parquet types --- python/cudf_polars/cudf_polars/dsl/utils/io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cudf_polars/cudf_polars/dsl/utils/io.py b/python/cudf_polars/cudf_polars/dsl/utils/io.py index 9693165dc1ae..634d3c9d2baa 100644 --- a/python/cudf_polars/cudf_polars/dsl/utils/io.py +++ b/python/cudf_polars/cudf_polars/dsl/utils/io.py @@ -138,7 +138,7 @@ def prefetch_parquet_file_metadata_for_ir( all_paths: set[str] = set() for node in traversal([root]): - if isinstance(node, StreamingScan): + if isinstance(node, StreamingScan) and node.base_scan.typ == "parquet": for scan in node.scans: for path in scan.paths: all_paths.add(path) @@ -197,7 +197,7 @@ def attach_cached_parquet_metadata( Mapping from file paths to cached parquet metadata. """ for node in traversal([root]): - if isinstance(node, StreamingScan): + if isinstance(node, StreamingScan) and node.base_scan.typ == "parquet": for scan in node.scans: cached = [cached_parquet_info_map[path] for path in scan.paths] Scan._validate_cached_parquet_info(scan.paths, cached) From c076a1ef558fea93a9a464b48fc749d84c02afa5 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Fri, 7 Aug 2026 12:56:02 -0700 Subject: [PATCH 07/26] Executor-dependent default --- python/cudf_polars/cudf_polars/utils/config.py | 7 +++++++ python/cudf_polars/tests/test_config.py | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/python/cudf_polars/cudf_polars/utils/config.py b/python/cudf_polars/cudf_polars/utils/config.py index 9c06469d5c58..5245a013f8f8 100644 --- a/python/cudf_polars/cudf_polars/utils/config.py +++ b/python/cudf_polars/cudf_polars/utils/config.py @@ -975,7 +975,14 @@ def from_polars_engine( if user_parquet_options is None: user_parquet_options = {} + user_parquet_options = dict(user_parquet_options) + if isinstance(user_parquet_options, dict): + # The default value of prefetch_file_metadata depends on the executor. + default_prefetch_file_metadata = user_executor == "streaming" + user_parquet_options.setdefault( + "prefetch_file_metadata", default_prefetch_file_metadata + ) parquet_options = ParquetOptions(**user_parquet_options) else: parquet_options = user_parquet_options diff --git a/python/cudf_polars/tests/test_config.py b/python/cudf_polars/tests/test_config.py index aa01207e7e51..2e5fef079e57 100644 --- a/python/cudf_polars/tests/test_config.py +++ b/python/cudf_polars/tests/test_config.py @@ -493,6 +493,13 @@ def test_validate_parquet_options(option: str) -> None: ) +def test_prefetch_file_metadata_default() -> None: + config = ConfigOptions.from_polars_engine(pl.GPUEngine(executor="streaming")) + assert config.parquet_options.prefetch_file_metadata is True + config = ConfigOptions.from_polars_engine(pl.GPUEngine(executor="in-memory")) + assert config.parquet_options.prefetch_file_metadata is False + + def test_prefetch_and_use_rapidsmpf_native_raises() -> None: with pytest.raises( NotImplementedError, From a8bb6fb3501188ab4dd1289574d9c0abbedb6013 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Fri, 7 Aug 2026 13:14:40 -0700 Subject: [PATCH 08/26] user option fixes --- python/cudf_polars/cudf_polars/utils/config.py | 17 ++++++++++------- python/cudf_polars/tests/test_config.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/python/cudf_polars/cudf_polars/utils/config.py b/python/cudf_polars/cudf_polars/utils/config.py index 5245a013f8f8..ead432220670 100644 --- a/python/cudf_polars/cudf_polars/utils/config.py +++ b/python/cudf_polars/cudf_polars/utils/config.py @@ -975,14 +975,17 @@ def from_polars_engine( if user_parquet_options is None: user_parquet_options = {} - user_parquet_options = dict(user_parquet_options) - if isinstance(user_parquet_options, dict): - # The default value of prefetch_file_metadata depends on the executor. - default_prefetch_file_metadata = user_executor == "streaming" - user_parquet_options.setdefault( - "prefetch_file_metadata", default_prefetch_file_metadata - ) + user_parquet_options = dict(user_parquet_options) + # Set the engine-dependent default, but don't override any user-provided values + # in-memory or via the environment. + if "prefetch_file_metadata" not in user_parquet_options and ( + os.environ.get(f"{ParquetOptions._env_prefix}__PREFETCH_FILE_METADATA") + is None + ): + user_parquet_options["prefetch_file_metadata"] = ( + user_executor == "streaming" + ) parquet_options = ParquetOptions(**user_parquet_options) else: parquet_options = user_parquet_options diff --git a/python/cudf_polars/tests/test_config.py b/python/cudf_polars/tests/test_config.py index 2e5fef079e57..36dd3cf101b0 100644 --- a/python/cudf_polars/tests/test_config.py +++ b/python/cudf_polars/tests/test_config.py @@ -33,6 +33,7 @@ InMemoryExecutor, JoinFilterPushdownOptions, MemoryResourceConfig, + ParquetOptions, StreamingExecutor, ) from cudf_polars.utils.cuda_stream import get_cuda_stream @@ -376,6 +377,13 @@ def test_parquet_options_from_env(monkeypatch: pytest.MonkeyPatch) -> None: assert config.parquet_options.prefetch_file_metadata is True assert config.parquet_options.use_jit_filter is True + with monkeypatch.context() as m: + # Env must win over the executor-derived default (streaming => True). + m.setenv("CUDF_POLARS__PARQUET_OPTIONS__PREFETCH_FILE_METADATA", "0") + engine = pl.GPUEngine(executor="streaming") + config = ConfigOptions.from_polars_engine(engine) + assert config.parquet_options.prefetch_file_metadata is False + with monkeypatch.context() as m: m.setenv("CUDF_POLARS__PARQUET_OPTIONS__CHUNKED", "foo") engine = pl.GPUEngine() @@ -500,6 +508,14 @@ def test_prefetch_file_metadata_default() -> None: assert config.parquet_options.prefetch_file_metadata is False +def test_parquet_options_object_passthrough() -> None: + parquet_options = ParquetOptions(prefetch_file_metadata=False) + config = ConfigOptions.from_polars_engine( + pl.GPUEngine(executor="streaming", parquet_options=parquet_options) + ) + assert config.parquet_options is parquet_options + + def test_prefetch_and_use_rapidsmpf_native_raises() -> None: with pytest.raises( NotImplementedError, From 5b6717fcb06618b8dd77a3c272d53e3befcf28e3 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Fri, 7 Aug 2026 13:18:30 -0700 Subject: [PATCH 09/26] fix --- python/pylibcudf/pylibcudf/io/parquet.pyx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/python/pylibcudf/pylibcudf/io/parquet.pyx b/python/pylibcudf/pylibcudf/io/parquet.pyx index 8f5a70a7dbc4..e892861e4fa0 100644 --- a/python/pylibcudf/pylibcudf/io/parquet.pyx +++ b/python/pylibcudf/pylibcudf/io/parquet.pyx @@ -628,8 +628,11 @@ cdef class ChunkedParquetReader: else: with nogil: sources = make_datasources(options.c_obj.get_source()) + # Pin wrappers for the nogil clone; do not rely on the caller's + # mutable parquet_metadatas container remaining unchanged. + metadata_holders = tuple(parquet_metadatas) metadata_ptrs = _parquet_metadata_ptrs( - parquet_metadatas, sources.size() + metadata_holders, sources.size() ) with nogil: c_metadatas = clone_parquet_metadatas( @@ -729,7 +732,10 @@ cpdef read_parquet( # Cython does not deep-copy vector[FileMetaData] while holding the GIL. with nogil: sources = make_datasources(options.c_obj.get_source()) - metadata_ptrs = _parquet_metadata_ptrs(parquet_metadatas, sources.size()) + # Pin wrappers for the nogil clone; do not rely on the caller's + # mutable parquet_metadatas container remaining unchanged. + metadata_holders = tuple(parquet_metadatas) + metadata_ptrs = _parquet_metadata_ptrs(metadata_holders, sources.size()) with nogil: c_metadatas = clone_parquet_metadatas( host_span[const_FileMetaData_ptr]( From 14f7258e20ca624257f0e02adcecf51330f3ea1f Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Fri, 7 Aug 2026 14:17:03 -0700 Subject: [PATCH 10/26] disable for use_rapidsmpf_native --- python/cudf_polars/tests/streaming/test_scan.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/cudf_polars/tests/streaming/test_scan.py b/python/cudf_polars/tests/streaming/test_scan.py index 9cb44ce6c0a7..0d45a7b5b6bb 100644 --- a/python/cudf_polars/tests/streaming/test_scan.py +++ b/python/cudf_polars/tests/streaming/test_scan.py @@ -96,7 +96,10 @@ def test_scan_parquet_use_rapidsmpf_native(tmp_path, df, streaming_engine_factor streaming_engine = streaming_engine_factory( StreamingOptions( target_partition_size=1_000, - parquet_options={"use_rapidsmpf_native": True}, + parquet_options={ + "use_rapidsmpf_native": True, + "prefetch_file_metadata": False, + }, ), ) make_partitioned_source(df, tmp_path, "parquet", n_files=1) From aebf8a00a73309ec354b9841aa0cba74cf8d0936 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Tue, 11 Aug 2026 01:22:29 +0000 Subject: [PATCH 11/26] revert commit 3: dont read page indices --- .../io/parquet/parquet_reader_metadata.cpp | 2 +- cpp/include/cudf/io/detail/parquet.hpp | 3 +- cpp/include/cudf/io/parquet.hpp | 3 +- cpp/include/cudf/io/parquet_metadata.hpp | 10 +--- cpp/src/io/functions.cpp | 4 +- cpp/src/io/parquet/reader_impl.cpp | 8 +-- cpp/src/io/parquet/reader_impl_helpers.hpp | 23 +-------- cpp/tests/io/parquet_reader_test.cpp | 51 ------------------- .../cudf_polars/cudf_polars/dsl/utils/io.py | 5 +- .../pylibcudf/io/parquet_metadata.pxd | 3 +- .../pylibcudf/io/parquet_metadata.pyi | 4 +- .../pylibcudf/io/parquet_metadata.pyx | 15 +----- .../pylibcudf/libcudf/io/parquet_metadata.pxd | 3 +- python/pylibcudf/tests/io/test_parquet.py | 4 +- 14 files changed, 20 insertions(+), 118 deletions(-) diff --git a/cpp/benchmarks/io/parquet/parquet_reader_metadata.cpp b/cpp/benchmarks/io/parquet/parquet_reader_metadata.cpp index 72fd2e9760d1..157442e20b5e 100644 --- a/cpp/benchmarks/io/parquet/parquet_reader_metadata.cpp +++ b/cpp/benchmarks/io/parquet/parquet_reader_metadata.cpp @@ -125,7 +125,7 @@ void BM_parquet_read_footer(nvbench::state& state) auto sources = cudf::io::make_datasources(source_info); timer.start(); - auto const metadatas = cudf::io::read_parquet_footers(sources, write_page_index); + auto const metadatas = cudf::io::read_parquet_footers(sources); timer.stop(); // Validate metadata diff --git a/cpp/include/cudf/io/detail/parquet.hpp b/cpp/include/cudf/io/detail/parquet.hpp index a9c4d7fda18d..41fc140629d9 100644 --- a/cpp/include/cudf/io/detail/parquet.hpp +++ b/cpp/include/cudf/io/detail/parquet.hpp @@ -260,12 +260,11 @@ parquet_metadata read_parquet_metadata(host_span con * @brief Constructs FileMetaData objects from parquet dataset * * @param sources Input `datasource` objects to read the dataset from - * @param read_page_indexes If true, deserialize page indexes into each column chunk when present * * @return List of FileMetaData objects, one per parquet source */ std::vector read_parquet_footers( - std::span const> sources, bool read_page_indexes = true); + std::span const> sources); /** * @brief Deep-copy FileMetaData objects from a span of pointers diff --git a/cpp/include/cudf/io/parquet.hpp b/cpp/include/cudf/io/parquet.hpp index b15f014e605b..bbb7766dff76 100644 --- a/cpp/include/cudf/io/parquet.hpp +++ b/cpp/include/cudf/io/parquet.hpp @@ -1018,8 +1018,7 @@ table_with_metadata read_parquet( * The following code snippet demonstrates how to read a dataset from a file: * @code * auto sources = cudf::io::make_datasources(cudf::io::source_info("dataset.parquet")); - * // Pass read_page_indexes=false when reusing only with read_parquet (page indexes optional). - * auto metadatas = cudf::io::read_parquet_footers(sources, false); + * auto metadatas = cudf::io::read_parquet_footers(sources); * auto options = cudf::io::parquet_reader_options::builder(); * auto result = cudf::io::read_parquet(std::move(sources), std::move(metadatas), options); * @endcode diff --git a/cpp/include/cudf/io/parquet_metadata.hpp b/cpp/include/cudf/io/parquet_metadata.hpp index 915ff6112b96..c6c0e6f4e376 100644 --- a/cpp/include/cudf/io/parquet_metadata.hpp +++ b/cpp/include/cudf/io/parquet_metadata.hpp @@ -285,22 +285,14 @@ parquet_metadata read_parquet_metadata(source_info const& src_info); /** * @brief Constructs FileMetaData objects from parquet dataset * - * By default (`read_page_indexes == true`), page indexes (`ColumnIndex` / `OffsetIndex`) are - * deserialized into each column chunk when present. Page indexes are not required by - * `read_parquet`; pass `false` when reusing footers only with `read_parquet` or - * `chunked_parquet_reader` to avoid the cost of materializing and later deep-cloning them. - * Hybrid-scan callers that need page-level stats can either keep the default or call - * `setup_page_index` separately. - * * @ingroup io_readers * * @param sources Input `datasource` objects to read the dataset from - * @param read_page_indexes If true, deserialize page indexes into each column chunk when present * * @return List of FileMetaData objects, one per parquet source */ std::vector read_parquet_footers( - std::span const> sources, bool read_page_indexes = true); + std::span const> sources); /** @} */ // end of group } // namespace io diff --git a/cpp/src/io/functions.cpp b/cpp/src/io/functions.cpp index 2b0a691d5a1e..5322cb40fc99 100644 --- a/cpp/src/io/functions.cpp +++ b/cpp/src/io/functions.cpp @@ -678,10 +678,10 @@ parquet_metadata read_parquet_metadata(source_info const& src_info) } std::vector read_parquet_footers( - std::span const> sources, bool read_page_indexes) + std::span const> sources) { CUDF_FUNC_RANGE(); - return detail_parquet::read_parquet_footers(sources, read_page_indexes); + return detail_parquet::read_parquet_footers(sources); } /** diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 0a44207c21ae..8fa305b309f9 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -1240,7 +1240,7 @@ parquet_metadata read_parquet_metadata(host_span con } std::vector read_parquet_footers( - std::span const> sources, bool read_page_indexes) + std::span const> sources) { // Do not use arrow schema when only reading the parquet metadata. constexpr auto use_arrow_schema = false; @@ -1248,8 +1248,10 @@ std::vector read_parquet_footers( // Do not select any columns when only reading the parquet metadata. constexpr auto has_column_projection = false; - // Parse the source dataset metadata. Callers that only reuse footers with read_parquet can - // pass read_page_indexes=false to skip materializing ColumnIndex/OffsetIndex (cheaper to clone). + // Read page indexes if available here since we will want to reuse the raw metadata for later use. + constexpr auto read_page_indexes = true; + + // Parse the source dataset metadata return aggregate_reader_metadata( host_span const>{sources.data(), sources.size()}, use_arrow_schema, diff --git a/cpp/src/io/parquet/reader_impl_helpers.hpp b/cpp/src/io/parquet/reader_impl_helpers.hpp index 59cfe5e65ee6..ed67ca05cfc3 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.hpp +++ b/cpp/src/io/parquet/reader_impl_helpers.hpp @@ -469,34 +469,15 @@ class aggregate_reader_metadata { std::span input_columns) const; /** - * @brief Get Parquet file metadatas (copies) + * @brief Get Parquet file metadatas * * @return Parquet file metadatas */ - [[nodiscard]] std::vector get_parquet_metadatas() const& + [[nodiscard]] std::vector get_parquet_metadatas() const { return std::vector{per_file_metadata.begin(), per_file_metadata.end()}; } - /** - * @brief Get Parquet file metadatas by moving out of this aggregate - * - * Used when the aggregate_reader_metadata is temporary and discarded after - * extracting footers (e.g. read_parquet_footers), avoiding a deep copy. - * - * @return Parquet file metadatas - */ - [[nodiscard]] std::vector get_parquet_metadatas() && - { - std::vector result; - result.reserve(per_file_metadata.size()); - for (auto& pfm : per_file_metadata) { - result.push_back(std::move(static_cast(pfm))); - } - per_file_metadata.clear(); - return result; - } - /** * @brief Extracts the schema_idx'th column chunk metadata from row_group_index'th row group of * the src_idx'th file. diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index d443a2d24e49..6ac067e74d3f 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -3159,57 +3159,6 @@ TEST_F(ParquetMetadataReaderTest, PreMaterializedMetadata) test_parquet_metadata(3); } -TEST_F(ParquetMetadataReaderTest, ReadParquetFootersPageIndexes) -{ - // Page indexes are only deserialized when BYTE_ARRAY columns are present. - auto const num_rows = 2000; - auto str_iter = cudf::detail::make_counting_transform_iterator( - 0, [](auto i) { return "str_" + std::to_string(i); }); - cudf::test::strings_column_wrapper str_col(str_iter, str_iter + num_rows); - table_view input_table({str_col}); - - auto filepath = temp_env->get_temp_filepath("ReadParquetFootersPageIndexes.parquet"); - cudf::io::parquet_writer_options out_opts = - cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, input_table) - .stats_level(cudf::io::statistics_freq::STATISTICS_COLUMN) - .row_group_size_rows(500) - .build(); - cudf::io::write_parquet(out_opts); - - auto const source_info = cudf::io::source_info{filepath}; - auto datasources = cudf::io::make_datasources(source_info); - - // Explicit false omits page indexes (lean path for read_parquet reuse). - { - auto metadatas = cudf::io::read_parquet_footers(datasources, false); - EXPECT_EQ(metadatas.size(), 1); - ASSERT_FALSE(metadatas.front().row_groups.empty()); - ASSERT_FALSE(metadatas.front().row_groups.front().columns.empty()); - EXPECT_FALSE(metadatas.front().row_groups.front().columns.front().offset_index.has_value()); - EXPECT_FALSE(metadatas.front().row_groups.front().columns.front().column_index.has_value()); - // File still records page-index byte ranges when written with column stats. - EXPECT_GT(metadatas.front().row_groups.front().columns.front().column_index_offset, 0); - EXPECT_GT(metadatas.front().row_groups.front().columns.front().column_index_length, 0); - - auto const options = cudf::io::parquet_reader_options::builder(source_info).build(); - auto sources_copy = cudf::io::make_datasources(source_info); - auto const read = - cudf::io::read_parquet(std::move(sources_copy), std::move(metadatas), options); - auto const expected = cudf::io::read_parquet(options); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected.tbl->view(), read.tbl->view()); - } - - // Default (true) materializes page indexes when present. - { - auto metadatas = cudf::io::read_parquet_footers(datasources); - EXPECT_EQ(metadatas.size(), 1); - ASSERT_FALSE(metadatas.front().row_groups.empty()); - ASSERT_FALSE(metadatas.front().row_groups.front().columns.empty()); - EXPECT_TRUE(metadatas.front().row_groups.front().columns.front().offset_index.has_value()); - EXPECT_TRUE(metadatas.front().row_groups.front().columns.front().column_index.has_value()); - } -} - TEST_F(ParquetMetadataReaderTest, Nested) { auto const num_rows = 1200; diff --git a/python/cudf_polars/cudf_polars/dsl/utils/io.py b/python/cudf_polars/cudf_polars/dsl/utils/io.py index 634d3c9d2baa..a76620aed835 100644 --- a/python/cudf_polars/cudf_polars/dsl/utils/io.py +++ b/python/cudf_polars/cudf_polars/dsl/utils/io.py @@ -90,16 +90,13 @@ def _prefetch_parquet_footers_for_paths(paths: list[str]) -> list[CachedParquetI else: sizes.append(None) - # Page indexes are not required by read_parquet; omit them to keep prefetched - # metadata lean for reuse/clone. metadata = plc.io.parquet_metadata.read_parquet_footers( plc.io.types.SourceInfo( [ plc.io.types.FilepathSource(path, size) for path, size in zip(paths, sizes, strict=True) ] - ), - read_page_indexes=False, + ) ) return [ diff --git a/python/pylibcudf/pylibcudf/io/parquet_metadata.pxd b/python/pylibcudf/pylibcudf/io/parquet_metadata.pxd index 634b856e5d1a..fedbca1800f2 100644 --- a/python/pylibcudf/pylibcudf/io/parquet_metadata.pxd +++ b/python/pylibcudf/pylibcudf/io/parquet_metadata.pxd @@ -1,7 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from libcpp cimport bool from libcpp.memory cimport unique_ptr from pylibcudf.io.types cimport SourceInfo @@ -98,4 +97,4 @@ cdef class RowGroup: cdef RowGroup from_cpp(cpp_RowGroup row_group) cpdef ParquetMetadata read_parquet_metadata(SourceInfo src_info) -cpdef list read_parquet_footers(SourceInfo src_info, bool read_page_indexes=*) +cpdef list read_parquet_footers(SourceInfo src_info) diff --git a/python/pylibcudf/pylibcudf/io/parquet_metadata.pyi b/python/pylibcudf/pylibcudf/io/parquet_metadata.pyi index 49f5d60f10fa..6aa9efb19713 100644 --- a/python/pylibcudf/pylibcudf/io/parquet_metadata.pyi +++ b/python/pylibcudf/pylibcudf/io/parquet_metadata.pyi @@ -110,6 +110,4 @@ class RowGroup: def ordinal(self) -> int | None: ... def read_parquet_metadata(src_info: SourceInfo) -> ParquetMetadata: ... -def read_parquet_footers( - src_info: SourceInfo, read_page_indexes: bool = True -) -> list[FileMetaData]: ... +def read_parquet_footers(src_info: SourceInfo) -> list[FileMetaData]: ... diff --git a/python/pylibcudf/pylibcudf/io/parquet_metadata.pyx b/python/pylibcudf/pylibcudf/io/parquet_metadata.pyx index 0f0386961c29..2042a6f2bd28 100644 --- a/python/pylibcudf/pylibcudf/io/parquet_metadata.pyx +++ b/python/pylibcudf/pylibcudf/io/parquet_metadata.pyx @@ -3,7 +3,6 @@ from cython.operator cimport dereference from libc.stdint cimport uint8_t -from libcpp cimport bool from libcpp.memory cimport make_unique, unique_ptr from libcpp.string cimport string from libcpp.utility cimport move @@ -650,22 +649,14 @@ cpdef ParquetMetadata read_parquet_metadata(SourceInfo src_info): return ParquetMetadata.from_metadata(c_result) -cpdef list read_parquet_footers(SourceInfo src_info, bool read_page_indexes=True): +cpdef list read_parquet_footers(SourceInfo src_info): """ Read parquet file footers as ``FileMetaData`` objects. - By default, page indexes (``ColumnIndex`` / ``OffsetIndex``) are deserialized - when present. They are not required by - :func:`~pylibcudf.io.parquet.read_parquet`; pass ``read_page_indexes=False`` - when reusing footers only with ``read_parquet`` or ``ChunkedParquetReader`` - to avoid materializing (and later deep-cloning) them. - Parameters ---------- src_info : SourceInfo Dataset source. - read_page_indexes : bool, default True - If True, deserialize page indexes into each column chunk when present. Returns ------- @@ -676,15 +667,13 @@ cpdef list read_parquet_footers(SourceInfo src_info, bool read_page_indexes=True cdef vector[cpp_FileMetaData] c_result cdef vector[unique_ptr[cpp_FileMetaData]] owned cdef size_t i, n - cdef bool c_read_page_indexes = read_page_indexes with nogil: sources = make_datasources(src_info.c_obj) c_result = cpp_parquet_metadata.read_parquet_footers( host_span[const_unique_ptr_datasource]( sources.data(), sources.size(), - ), - c_read_page_indexes, + ) ) n = c_result.size() owned.reserve(n) diff --git a/python/pylibcudf/pylibcudf/libcudf/io/parquet_metadata.pxd b/python/pylibcudf/pylibcudf/libcudf/io/parquet_metadata.pxd index f0b7aec61a26..758207e4909f 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/parquet_metadata.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/parquet_metadata.pxd @@ -46,6 +46,5 @@ cdef extern from "cudf/io/parquet_metadata.hpp" namespace "cudf::io" nogil: ) except +libcudf_exception_handler cdef vector[FileMetaData] read_parquet_footers( - host_span[const_unique_ptr_datasource] sources, - bool read_page_indexes + host_span[const_unique_ptr_datasource] sources ) except +libcudf_exception_handler diff --git a/python/pylibcudf/tests/io/test_parquet.py b/python/pylibcudf/tests/io/test_parquet.py index 5cdc2540ac5d..be56621c2bc9 100644 --- a/python/pylibcudf/tests/io/test_parquet.py +++ b/python/pylibcudf/tests/io/test_parquet.py @@ -318,11 +318,9 @@ def test_read_parquet_from_device_buffers( assert_table_and_meta_eq(expected, res, check_field_nullability=False) -@pytest.mark.parametrize("read_page_indexes", [False, True]) def test_read_parquet_with_pre_materialized_metadata( table_data: tuple[plc.io.types.TableWithMetadata, pa.Table], binary_source_or_sink: str | os.PathLike[str] | io.BytesIO, - read_page_indexes: bool, ) -> None: _, pa_table = table_data source = make_source( @@ -332,7 +330,7 @@ def test_read_parquet_with_pre_materialized_metadata( options = plc.io.parquet.ParquetReaderOptions.builder(source_info).build() parquet_metadatas = plc.io.parquet_metadata.read_parquet_footers( - source_info, read_page_indexes=read_page_indexes + source_info ) result = plc.io.parquet.read_parquet( options, parquet_metadatas=parquet_metadatas From f9490ac7455e40d1cac803c438b465eb17419018 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Tue, 11 Aug 2026 19:49:49 +0000 Subject: [PATCH 12/26] move helper function from C++ to Cython (verbatim) --- cpp/include/cudf/io/detail/parquet.hpp | 13 ------------ cpp/src/io/parquet/reader_impl.cpp | 14 ------------- python/pylibcudf/pylibcudf/io/parquet.pyx | 6 +++--- .../pylibcudf/libcudf/io/parquet.pxd | 20 ++++++++++++++++--- 4 files changed, 20 insertions(+), 33 deletions(-) diff --git a/cpp/include/cudf/io/detail/parquet.hpp b/cpp/include/cudf/io/detail/parquet.hpp index 41fc140629d9..3ec2090ad032 100644 --- a/cpp/include/cudf/io/detail/parquet.hpp +++ b/cpp/include/cudf/io/detail/parquet.hpp @@ -266,19 +266,6 @@ parquet_metadata read_parquet_metadata(host_span con std::vector read_parquet_footers( std::span const> sources); -/** - * @brief Deep-copy FileMetaData objects from a span of pointers - * - * Used when handing cached footers into the parquet reader so the clone can - * run without the Python GIL (Cython calls this under `nogil`). - * - * @param sources Non-owning pointers to FileMetaData objects to clone - * - * @return Deep copies of each pointed-to FileMetaData - */ -[[nodiscard]] std::vector clone_parquet_metadatas( - host_span sources); - } // namespace parquet::detail } // namespace io } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 8fa305b309f9..75394523b594 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -1260,18 +1260,4 @@ std::vector read_parquet_footers( .get_parquet_metadatas(); } -[[nodiscard]] std::vector clone_parquet_metadatas( - host_span sources) -{ - CUDF_FUNC_RANGE(); - - std::vector result; - result.reserve(sources.size()); - for (auto const* src : sources) { - CUDF_EXPECTS(src != nullptr, "Null FileMetaData pointer"); - result.push_back(*src); - } - return result; -} - } // namespace cudf::io::parquet::detail diff --git a/python/pylibcudf/pylibcudf/io/parquet.pyx b/python/pylibcudf/pylibcudf/io/parquet.pyx index e892861e4fa0..85bce1d038a0 100644 --- a/python/pylibcudf/pylibcudf/io/parquet.pyx +++ b/python/pylibcudf/pylibcudf/io/parquet.pyx @@ -28,7 +28,7 @@ from pylibcudf.libcudf.expressions cimport expression from pylibcudf.libcudf.io.datasource cimport datasource, make_datasources from pylibcudf.libcudf.io.parquet cimport ( chunked_parquet_reader as cpp_chunked_parquet_reader, - clone_parquet_metadatas, + copy_parquet_metadatas, const_FileMetaData_ptr, parquet_reader_options, read_parquet as cpp_read_parquet, @@ -635,7 +635,7 @@ cdef class ChunkedParquetReader: metadata_holders, sources.size() ) with nogil: - c_metadatas = clone_parquet_metadatas( + c_metadatas = copy_parquet_metadatas( host_span[const_FileMetaData_ptr]( metadata_ptrs.data(), metadata_ptrs.size(), @@ -737,7 +737,7 @@ cpdef read_parquet( metadata_holders = tuple(parquet_metadatas) metadata_ptrs = _parquet_metadata_ptrs(metadata_holders, sources.size()) with nogil: - c_metadatas = clone_parquet_metadatas( + c_metadatas = copy_parquet_metadatas( host_span[const_FileMetaData_ptr]( metadata_ptrs.data(), metadata_ptrs.size(), diff --git a/python/pylibcudf/pylibcudf/libcudf/io/parquet.pxd b/python/pylibcudf/pylibcudf/libcudf/io/parquet.pxd index 3a922e90790d..184c532b2d46 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/parquet.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/parquet.pxd @@ -356,7 +356,21 @@ from pylibcudf.libcudf.utilities.span cimport host_span ctypedef const FileMetaData* const_FileMetaData_ptr -cdef extern from "cudf/io/detail/parquet.hpp" namespace "cudf::io::parquet::detail" nogil: - cdef vector[FileMetaData] clone_parquet_metadatas( +cdef extern from *: + """ + #include + #include + #include + + std::vector copy_parquet_metadatas( + cudf::host_span sources) + { + std::vector result; + result.reserve(sources.size()); + for (auto const* src : sources) { result.push_back(*src); } + return result; + } + """ + cdef vector[FileMetaData] copy_parquet_metadatas( host_span[const_FileMetaData_ptr] sources - ) except +libcudf_exception_handler + ) except +libcudf_exception_handler nogil From 5e864acff4afaa693f6a28157b85e7d9a347dbb6 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Tue, 11 Aug 2026 20:30:50 +0000 Subject: [PATCH 13/26] properly handle unspecified prefetch meta in parquet options --- .../cudf_polars/cudf_polars/engine/options.py | 34 +--------- .../cudf_polars/cudf_polars/utils/config.py | 68 ++++++++++++++++--- 2 files changed, 58 insertions(+), 44 deletions(-) diff --git a/python/cudf_polars/cudf_polars/engine/options.py b/python/cudf_polars/cudf_polars/engine/options.py index 611129cf2c01..22cbab1731e8 100644 --- a/python/cudf_polars/cudf_polars/engine/options.py +++ b/python/cudf_polars/cudf_polars/engine/options.py @@ -18,7 +18,7 @@ from cudf_polars.engine.hardware_binding import ( HardwareBindingPolicy, ) -from cudf_polars.utils.config import MemoryResourceConfig +from cudf_polars.utils.config import UNSPECIFIED, MemoryResourceConfig, Unspecified if TYPE_CHECKING: from collections.abc import Callable @@ -37,38 +37,6 @@ ] -class Unspecified: - """ - Sentinel value meaning "fall back to environment variable, then built-in default". - - The singleton instance :data:`UNSPECIFIED` is used as the default for every - :class:`StreamingOptions` field. When a field is still ``UNSPECIFIED`` after - construction (i.e. neither an explicit value nor an environment variable was provided), - the underlying library applies its own built-in default. - """ - - _instance: Unspecified | None = None - - def __new__(cls) -> Unspecified: - """Return the singleton instance.""" - if cls._instance is None: - cls._instance = super().__new__(cls) - return cls._instance - - def __repr__(self) -> str: - """Return ``"UNSPECIFIED"``.""" - return "UNSPECIFIED" - - -UNSPECIFIED = Unspecified() -"""Singleton sentinel for all :class:`StreamingOptions` fields. - -A field set to ``UNSPECIFIED`` after construction means no explicit value and no -matching environment variable was found; the underlying library will apply its own -built-in default. -""" - - def _opt( category: str, env_var: str | None = None, diff --git a/python/cudf_polars/cudf_polars/utils/config.py b/python/cudf_polars/cudf_polars/utils/config.py index 2b0c988c941f..2376957ca231 100644 --- a/python/cudf_polars/cudf_polars/utils/config.py +++ b/python/cudf_polars/cudf_polars/utils/config.py @@ -49,6 +49,7 @@ __all__ = [ + "UNSPECIFIED", "Cluster", "ConfigOptions", "DaskContext", @@ -60,9 +61,42 @@ "SPMDContext", "StreamingExecutor", "StreamingFallbackMode", + "Unspecified", ] +class Unspecified: + """ + Sentinel value meaning "fall back to environment variable, then built-in default". + + The singleton instance :data:`UNSPECIFIED` is used as the default for every + :class:`StreamingOptions` field. When a field is still ``UNSPECIFIED`` after + construction (i.e. neither an explicit value nor an environment variable was provided), + the underlying library applies its own built-in default. + """ + + _instance: Unspecified | None = None + + def __new__(cls) -> Unspecified: + """Return the singleton instance.""" + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __repr__(self) -> str: + """Return ``"UNSPECIFIED"``.""" + return "UNSPECIFIED" + + +UNSPECIFIED = Unspecified() +"""Singleton sentinel for all :class:`StreamingOptions` fields. + +A field set to ``UNSPECIFIED`` after construction means no explicit value and no +matching environment variable was found; the underlying library will apply its own +built-in default. +""" + + def _env_get_int(name: str, default: int) -> int: try: return int(os.getenv(name, default)) @@ -261,11 +295,11 @@ class ParquetOptions: f"{_env_prefix}__MAX_ROW_GROUP_SAMPLES", int, default=1 ) ) - prefetch_file_metadata: bool = dataclasses.field( + prefetch_file_metadata: bool | Unspecified = dataclasses.field( default_factory=_make_default_factory( f"{_env_prefix}__PREFETCH_FILE_METADATA", _bool_converter, - default=True, + default=UNSPECIFIED, ) ) use_jit_filter: bool = dataclasses.field( @@ -289,7 +323,7 @@ def __post_init__(self) -> None: # noqa: D105 raise TypeError("max_footer_samples must be an int") if not isinstance(self.max_row_group_samples, int): raise TypeError("max_row_group_samples must be an int") - if not isinstance(self.prefetch_file_metadata, bool): + if not isinstance(self.prefetch_file_metadata, (bool, Unspecified)): raise TypeError("prefetch_file_metadata must be a bool") if not isinstance(self.use_jit_filter, bool): raise TypeError("use_jit_filter must be a bool") @@ -960,19 +994,31 @@ def from_polars_engine( if user_parquet_options is None: user_parquet_options = {} + # Engine-dependent default: only prefetch for the streaming executor. + # Skipped if the user or the environment has already set a value. + prefetch_default = user_executor == "streaming" + prefetch_env_set = ( + os.environ.get(f"{ParquetOptions._env_prefix}__PREFETCH_FILE_METADATA") + is not None + ) + if isinstance(user_parquet_options, dict): user_parquet_options = dict(user_parquet_options) - # Set the engine-dependent default, but don't override any user-provided values - # in-memory or via the environment. - if "prefetch_file_metadata" not in user_parquet_options and ( - os.environ.get(f"{ParquetOptions._env_prefix}__PREFETCH_FILE_METADATA") - is None + if ( + "prefetch_file_metadata" not in user_parquet_options + and not prefetch_env_set ): - user_parquet_options["prefetch_file_metadata"] = ( - user_executor == "streaming" - ) + user_parquet_options["prefetch_file_metadata"] = prefetch_default parquet_options = ParquetOptions(**user_parquet_options) else: + if ( + isinstance(user_parquet_options.prefetch_file_metadata, Unspecified) + and not prefetch_env_set + ): + user_parquet_options = dataclasses.replace( + user_parquet_options, + prefetch_file_metadata=prefetch_default, + ) parquet_options = user_parquet_options # This is set in polars, and so can't be overridden by the environment user_raise_on_fail = engine.config.get("raise_on_fail", False) From 3c1d78bb2b132dfcdbd0ba73591c6674e85c6007 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Wed, 12 Aug 2026 15:15:23 +0000 Subject: [PATCH 14/26] turn on for cloud-only --- python/cudf_polars/cudf_polars/dsl/ir.py | 13 ++----------- python/cudf_polars/cudf_polars/dsl/utils/io.py | 13 ++++++++++++- python/cudf_polars/cudf_polars/engine/core.py | 6 ++++-- python/cudf_polars/cudf_polars/utils/config.py | 2 +- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/python/cudf_polars/cudf_polars/dsl/ir.py b/python/cudf_polars/cudf_polars/dsl/ir.py index 559777f43c7d..7e7ebb6785d3 100644 --- a/python/cudf_polars/cudf_polars/dsl/ir.py +++ b/python/cudf_polars/cudf_polars/dsl/ir.py @@ -902,12 +902,7 @@ def _get_parquet_row_count_from_metadata( ) -> int: # Zero-width parquet files lose their row count when read through # pylibcudf. See https://github.com/rapidsai/cudf/issues/21428 - if parquet_options.prefetch_file_metadata: - if cached_parquet_info is None: - raise AssertionError( - "Cached parquet info is required when prefetching file metadata is enabled" - ) - + if cached_parquet_info is not None: Scan._validate_cached_parquet_info(paths, cached_parquet_info) parquet_metadatas = [ info.file_metadata for info in cached_parquet_info @@ -1071,11 +1066,7 @@ def read_csv_header( df, ) elif typ == "parquet": - if parquet_options.prefetch_file_metadata: - if cached_parquet_info is None: - raise AssertionError( - "Cached parquet info is required when prefetching file metadata is enabled" - ) + if cached_parquet_info is not None: Scan._validate_cached_parquet_info(paths, cached_parquet_info) filepath_sources = [] parquet_metadatas = [] diff --git a/python/cudf_polars/cudf_polars/dsl/utils/io.py b/python/cudf_polars/cudf_polars/dsl/utils/io.py index a76620aed835..a4ed9eac758f 100644 --- a/python/cudf_polars/cudf_polars/dsl/utils/io.py +++ b/python/cudf_polars/cudf_polars/dsl/utils/io.py @@ -110,6 +110,8 @@ def prefetch_parquet_file_metadata_for_ir( root: IR, py_executor: concurrent.futures.Executor | None, stats: StatsCollector | None = None, + *, + remote_only: bool = False, ) -> dict[str, CachedParquetInfo]: """ Prefetch parquet metadata for all parquet scans in an IR graph. @@ -125,6 +127,9 @@ def prefetch_parquet_file_metadata_for_ir( prefetched during statistics collection, when the number of files sampled equals the total number of files. Providing ``stats`` here will skip rereading metadata for those files. + remote_only + If ``True``, only prefetch metadata for remote URIs (e.g. ``s3://``), + skipping local paths. Returns ------- @@ -155,6 +160,10 @@ def prefetch_parquet_file_metadata_for_ir( cached_parquet_info[info.path] = info missing_paths = all_paths - set(cached_parquet_info.keys()) + if remote_only: + missing_paths = { + p for p in missing_paths if plc.io.SourceInfo._is_remote_uri(p) + } cm: contextlib.AbstractContextManager[concurrent.futures.Executor | None] if py_executor is None: @@ -196,7 +205,9 @@ def attach_cached_parquet_metadata( for node in traversal([root]): if isinstance(node, StreamingScan) and node.base_scan.typ == "parquet": for scan in node.scans: - cached = [cached_parquet_info_map[path] for path in scan.paths] + cached = [cached_parquet_info_map.get(path) for path in scan.paths] + if any(info is None for info in cached): + continue Scan._validate_cached_parquet_info(scan.paths, cached) scan.cached_parquet_info = cached scan._non_child_args = (*scan._non_child_args[:-1], cached) diff --git a/python/cudf_polars/cudf_polars/engine/core.py b/python/cudf_polars/cudf_polars/engine/core.py index b382f12099a4..771ec1f22177 100644 --- a/python/cudf_polars/cudf_polars/engine/core.py +++ b/python/cudf_polars/cudf_polars/engine/core.py @@ -42,7 +42,7 @@ from cudf_polars.streaming.parallel import lower_ir_graph_with_node_map from cudf_polars.streaming.statistics import collect_statistics from cudf_polars.streaming.utils import _concat -from cudf_polars.utils.config import get_total_device_memory +from cudf_polars.utils.config import Unspecified, get_total_device_memory if TYPE_CHECKING: from collections.abc import Callable, MutableMapping @@ -778,11 +778,13 @@ def evaluate_on_rank( py_executor, get_cuda_stream=ctx.br().stream_pool.get_stream, query_id=query_id ) - if config_options.parquet_options.prefetch_file_metadata: + prefetch_file_metadata = config_options.parquet_options.prefetch_file_metadata + if prefetch_file_metadata is not False: cached_parquet_info_map = prefetch_parquet_file_metadata_for_ir( ir, ir_context.py_executor, stats=stats, + remote_only=isinstance(prefetch_file_metadata, Unspecified), ) attach_cached_parquet_metadata(ir, cached_parquet_info_map) diff --git a/python/cudf_polars/cudf_polars/utils/config.py b/python/cudf_polars/cudf_polars/utils/config.py index 2376957ca231..d0415bad33a4 100644 --- a/python/cudf_polars/cudf_polars/utils/config.py +++ b/python/cudf_polars/cudf_polars/utils/config.py @@ -996,7 +996,7 @@ def from_polars_engine( # Engine-dependent default: only prefetch for the streaming executor. # Skipped if the user or the environment has already set a value. - prefetch_default = user_executor == "streaming" + prefetch_default = UNSPECIFIED if user_executor == "streaming" else False prefetch_env_set = ( os.environ.get(f"{ParquetOptions._env_prefix}__PREFETCH_FILE_METADATA") is not None From 93bccd7bf6ff05acd7d7fd4abd5b65b3cd681809 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Wed, 12 Aug 2026 16:48:33 +0000 Subject: [PATCH 15/26] pre-commit --- python/cudf_polars/cudf_polars/dsl/utils/io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cudf_polars/cudf_polars/dsl/utils/io.py b/python/cudf_polars/cudf_polars/dsl/utils/io.py index a4ed9eac758f..3ac9db651b01 100644 --- a/python/cudf_polars/cudf_polars/dsl/utils/io.py +++ b/python/cudf_polars/cudf_polars/dsl/utils/io.py @@ -205,9 +205,9 @@ def attach_cached_parquet_metadata( for node in traversal([root]): if isinstance(node, StreamingScan) and node.base_scan.typ == "parquet": for scan in node.scans: - cached = [cached_parquet_info_map.get(path) for path in scan.paths] - if any(info is None for info in cached): + if not all(path in cached_parquet_info_map for path in scan.paths): continue + cached = [cached_parquet_info_map[path] for path in scan.paths] Scan._validate_cached_parquet_info(scan.paths, cached) scan.cached_parquet_info = cached scan._non_child_args = (*scan._non_child_args[:-1], cached) From df0b4d418a726715868acba005f6f84e3e8325d5 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Wed, 12 Aug 2026 17:10:20 +0000 Subject: [PATCH 16/26] remove copyright change --- cpp/benchmarks/io/parquet/parquet_reader_metadata.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/benchmarks/io/parquet/parquet_reader_metadata.cpp b/cpp/benchmarks/io/parquet/parquet_reader_metadata.cpp index 157442e20b5e..28f5ada2adf8 100644 --- a/cpp/benchmarks/io/parquet/parquet_reader_metadata.cpp +++ b/cpp/benchmarks/io/parquet/parquet_reader_metadata.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ From ad20e44601798d06d167d2833cac63951f4f2a9b Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Thu, 13 Aug 2026 13:42:22 +0000 Subject: [PATCH 17/26] default is Unspecified for prefetching with streaming engine --- python/cudf_polars/tests/test_config.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/python/cudf_polars/tests/test_config.py b/python/cudf_polars/tests/test_config.py index dd6260049e9a..7dad7455fc94 100644 --- a/python/cudf_polars/tests/test_config.py +++ b/python/cudf_polars/tests/test_config.py @@ -35,6 +35,7 @@ MemoryResourceConfig, ParquetOptions, StreamingExecutor, + Unspecified, ) from cudf_polars.utils.cuda_stream import get_cuda_stream @@ -502,10 +503,18 @@ def test_validate_parquet_options(option: str) -> None: def test_prefetch_file_metadata_default() -> None: config = ConfigOptions.from_polars_engine(pl.GPUEngine(executor="streaming")) - assert config.parquet_options.prefetch_file_metadata is True + assert isinstance(config.parquet_options.prefetch_file_metadata, Unspecified) + config = ConfigOptions.from_polars_engine(pl.GPUEngine(executor="in-memory")) assert config.parquet_options.prefetch_file_metadata is False + config = ConfigOptions.from_polars_engine( + pl.GPUEngine( + executor="streaming", parquet_options={"prefetch_file_metadata": True} + ) + ) + assert config.parquet_options.prefetch_file_metadata is True + def test_parquet_options_object_passthrough() -> None: parquet_options = ParquetOptions(prefetch_file_metadata=False) From 74cda45d46b8039f9e291884a1e68f21c31571e7 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Thu, 13 Aug 2026 13:58:17 +0000 Subject: [PATCH 18/26] add a test for ParquetOptions w/o setting prefetching default --- python/cudf_polars/tests/test_config.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/python/cudf_polars/tests/test_config.py b/python/cudf_polars/tests/test_config.py index 7dad7455fc94..883d5cd06374 100644 --- a/python/cudf_polars/tests/test_config.py +++ b/python/cudf_polars/tests/test_config.py @@ -524,6 +524,24 @@ def test_parquet_options_object_passthrough() -> None: assert config.parquet_options is parquet_options +def test_parquet_options_object_engine_default() -> None: + # If a user passes in a ParquetOptions object instead of a plain dict, and + # doesn't set prefetch_file_metadata on it, we still need to fill in the + # right default for the chosen executor. + parquet_options = ParquetOptions() + assert isinstance(parquet_options.prefetch_file_metadata, Unspecified) + + config = ConfigOptions.from_polars_engine( + pl.GPUEngine(executor="in-memory", parquet_options=parquet_options) + ) + assert config.parquet_options.prefetch_file_metadata is False + + config = ConfigOptions.from_polars_engine( + pl.GPUEngine(executor="streaming", parquet_options=parquet_options) + ) + assert isinstance(config.parquet_options.prefetch_file_metadata, Unspecified) + + def test_validate_raise_on_fail() -> None: with pytest.raises(TypeError, match="'raise_on_fail' must be"): ConfigOptions.from_polars_engine( From 0af3e327280188d021452254ac4a34a51325973f Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Thu, 13 Aug 2026 14:08:43 +0000 Subject: [PATCH 19/26] fix tests because we fallback now if metadata is not prefetched --- .../cudf_polars/tests/streaming/test_scan.py | 22 +------------- python/cudf_polars/tests/test_select.py | 29 ++++++++++++------- 2 files changed, 20 insertions(+), 31 deletions(-) diff --git a/python/cudf_polars/tests/streaming/test_scan.py b/python/cudf_polars/tests/streaming/test_scan.py index 0fcf3c663a9f..972bf00a9bf1 100644 --- a/python/cudf_polars/tests/streaming/test_scan.py +++ b/python/cudf_polars/tests/streaming/test_scan.py @@ -349,33 +349,13 @@ def test_streaming_scan_raises() -> None: StreamingScan.do_evaluate([fused], scan, context=ctx) -def test_scan_missing_prefetch_metadata_raises() -> None: +def test_scan_path_mismatch_raises() -> None: # This isn't reachable by polars' public API, so we test it directly. scan = _make_parquet_scan( ["file.parquet"], parquet_options=ParquetOptions(prefetch_file_metadata=True) ) ctx = IRExecutionContext() - with pytest.raises( - AssertionError, - match=r"Cached parquet info is required", - ): - Scan.do_evaluate( - scan.schema, - scan.typ, - scan.reader_options, - scan.paths, - scan.with_columns, - scan.skip_rows, - scan.n_rows, - scan.row_index, - scan.include_file_paths, - scan.predicate, - scan.parquet_options, - None, - context=ctx, - ) - with pytest.raises( AssertionError, match=r"Paths do not match cached parquet info", diff --git a/python/cudf_polars/tests/test_select.py b/python/cudf_polars/tests/test_select.py index 435bec5d2031..07405d0d670e 100644 --- a/python/cudf_polars/tests/test_select.py +++ b/python/cudf_polars/tests/test_select.py @@ -176,18 +176,27 @@ def parquet_scan_row_bounds(request) -> dict[str, int | None]: return request.param -def test_get_parquet_row_count_from_metadata_raises() -> None: - paths = ["/some/missing/file.parquet"] +def test_get_parquet_row_count_from_metadata_no_cache_falls_back(tmp_path) -> None: + # If no cached parquet info is available (e.g. because prefetching was + # skipped for this path), we fall back to reading the metadata directly, + # rather than raising. + source = tmp_path / "data.parquet" + pl.DataFrame({"a": range(5)}).write_parquet(source) parquet_options = ParquetOptions(prefetch_file_metadata=True) - with pytest.raises(AssertionError, match=r"Cached parquet info is required"): - Scan._get_parquet_row_count_from_metadata( - paths, - skip_rows=0, - n_rows=-1, - parquet_options=parquet_options, - cached_parquet_info=None, - ) + row_count = Scan._get_parquet_row_count_from_metadata( + [str(source)], + skip_rows=0, + n_rows=-1, + parquet_options=parquet_options, + cached_parquet_info=None, + ) + assert row_count == 5 + + +def test_get_parquet_row_count_from_metadata_path_mismatch_raises() -> None: + paths = ["/some/missing/file.parquet"] + parquet_options = ParquetOptions(prefetch_file_metadata=True) with pytest.raises( AssertionError, From 4c5510820db4fadc1a96738711257aacd6a56d1e Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Thu, 13 Aug 2026 14:12:12 +0000 Subject: [PATCH 20/26] test remote_only=True --- .../cudf_polars/tests/streaming/test_scan.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/python/cudf_polars/tests/streaming/test_scan.py b/python/cudf_polars/tests/streaming/test_scan.py index 972bf00a9bf1..a44af628ea1c 100644 --- a/python/cudf_polars/tests/streaming/test_scan.py +++ b/python/cudf_polars/tests/streaming/test_scan.py @@ -126,6 +126,27 @@ def test_prefetch_parquet_file_metadata_no_parquet_scans() -> None: assert result == {} +def test_prefetch_parquet_file_metadata_remote_only(tmp_path, df) -> None: + make_partitioned_source(df, tmp_path, "parquet", n_files=1) + local_path = str(next(tmp_path.glob("*.parquet"))) + + scan = _make_parquet_scan([local_path]) + fused = FusedScan(scan.schema, scan, scan.paths, scan.parquet_options, []) + streaming_scan = StreamingScan([fused], scan, "fused") + + # Local paths are skipped entirely when remote_only=True. + result = prefetch_parquet_file_metadata_for_ir( + streaming_scan, py_executor=None, stats=None, remote_only=True + ) + assert result == {} + + # The same local path is prefetched when remote_only=False (the default). + result = prefetch_parquet_file_metadata_for_ir( + streaming_scan, py_executor=None, stats=None + ) + assert set(result) == {local_path} + + def test_prefetch_file_metadata_select_fast_count( df: pl.DataFrame, streaming_engine_factory: Callable[..., StreamingEngine], From 071bd048c681997a36e1a419140ff0c6715c3a91 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Thu, 13 Aug 2026 14:17:23 +0000 Subject: [PATCH 21/26] docs, small fixes --- python/cudf_polars/cudf_polars/utils/config.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/python/cudf_polars/cudf_polars/utils/config.py b/python/cudf_polars/cudf_polars/utils/config.py index d0415bad33a4..e4d4461687d2 100644 --- a/python/cudf_polars/cudf_polars/utils/config.py +++ b/python/cudf_polars/cudf_polars/utils/config.py @@ -255,7 +255,10 @@ class ParquetOptions: will also be skipped if ``max_footer_samples`` is 0. prefetch_file_metadata Whether to prefetch parquet file metadata and pass it through - `parquet_metadatas` to avoid rereading file footers. Enabled by default. + `parquet_metadatas` to avoid rereading file footers. Not supported + by the in-memory executor, where it defaults to disabled. For the + streaming executor, it defaults to being enabled for remote URIs + (e.g. ``s3://``) only; pass ``True`` to also prefetch local files. use_jit_filter Whether to use JIT compilation for post-read filtering in Parquet scans. When enabled, filter predicates are JIT-compiled to CUDA kernels for @@ -324,7 +327,7 @@ def __post_init__(self) -> None: # noqa: D105 if not isinstance(self.max_row_group_samples, int): raise TypeError("max_row_group_samples must be an int") if not isinstance(self.prefetch_file_metadata, (bool, Unspecified)): - raise TypeError("prefetch_file_metadata must be a bool") + raise TypeError("prefetch_file_metadata must be a bool when specified") if not isinstance(self.use_jit_filter, bool): raise TypeError("use_jit_filter must be a bool") @@ -1047,7 +1050,7 @@ def from_polars_engine( match user_executor: case "in-memory": executor = InMemoryExecutor(**user_executor_options) - if parquet_options.prefetch_file_metadata: + if parquet_options.prefetch_file_metadata is True: raise NotImplementedError( "Prefetching is not supported for the in-memory executor." ) From ea351a6325517fbe57de754a4c52a84a31dd003a Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Thu, 13 Aug 2026 14:21:06 +0000 Subject: [PATCH 22/26] revert unnecessary changes --- python/pylibcudf/pylibcudf/libcudf/io/parquet_metadata.pxd | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/pylibcudf/pylibcudf/libcudf/io/parquet_metadata.pxd b/python/pylibcudf/pylibcudf/libcudf/io/parquet_metadata.pxd index 758207e4909f..3a2b41de2774 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/parquet_metadata.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/parquet_metadata.pxd @@ -1,7 +1,6 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 from libc.stdint cimport int64_t -from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.string cimport string from libcpp.unordered_map cimport unordered_map From d84b230a0251aacec71ca38be4c1df5c4f805711 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Thu, 13 Aug 2026 14:35:06 +0000 Subject: [PATCH 23/26] remove copy_parquet_metadatas --- python/pylibcudf/pylibcudf/io/parquet.pyx | 23 +++++++------------ .../pylibcudf/libcudf/io/parquet.pxd | 23 ------------------- 2 files changed, 8 insertions(+), 38 deletions(-) diff --git a/python/pylibcudf/pylibcudf/io/parquet.pyx b/python/pylibcudf/pylibcudf/io/parquet.pyx index 85bce1d038a0..b752d1a442f6 100644 --- a/python/pylibcudf/pylibcudf/io/parquet.pyx +++ b/python/pylibcudf/pylibcudf/io/parquet.pyx @@ -28,8 +28,6 @@ from pylibcudf.libcudf.expressions cimport expression from pylibcudf.libcudf.io.datasource cimport datasource, make_datasources from pylibcudf.libcudf.io.parquet cimport ( chunked_parquet_reader as cpp_chunked_parquet_reader, - copy_parquet_metadatas, - const_FileMetaData_ptr, parquet_reader_options, read_parquet as cpp_read_parquet, write_parquet as cpp_write_parquet, @@ -40,7 +38,6 @@ from pylibcudf.libcudf.io.parquet cimport ( chunked_parquet_writer_options, merge_row_group_metadata as cpp_merge_row_group_metadata, ) -from pylibcudf.libcudf.utilities.span cimport host_span from pylibcudf.libcudf.io.parquet_schema cimport FileMetaData as cpp_FileMetaData from pylibcudf.libcudf.io.types cimport ( compression_type, @@ -614,6 +611,7 @@ cdef class ChunkedParquetReader: cdef vector[cpp_FileMetaData] c_metadatas cdef vector[cpp_FileMetaData*] metadata_ptrs cdef cudaStream_t stream_view = self._stream.view().value() + cdef size_t i if parquet_metadatas is None: with nogil: self.reader.reset( @@ -635,12 +633,9 @@ cdef class ChunkedParquetReader: metadata_holders, sources.size() ) with nogil: - c_metadatas = copy_parquet_metadatas( - host_span[const_FileMetaData_ptr]( - metadata_ptrs.data(), - metadata_ptrs.size(), - ) - ) + c_metadatas.reserve(metadata_ptrs.size()) + for i in range(metadata_ptrs.size()): + c_metadatas.push_back(dereference(metadata_ptrs[i])) self.reader.reset( new cpp_chunked_parquet_reader( chunk_read_limit, @@ -723,6 +718,7 @@ cpdef read_parquet( cdef vector[cpp_FileMetaData] c_metadatas cdef vector[cpp_FileMetaData*] metadata_ptrs cdef table_with_metadata c_result + cdef size_t i mr = _get_memory_resource(mr) if parquet_metadatas is None: with nogil: @@ -737,12 +733,9 @@ cpdef read_parquet( metadata_holders = tuple(parquet_metadatas) metadata_ptrs = _parquet_metadata_ptrs(metadata_holders, sources.size()) with nogil: - c_metadatas = copy_parquet_metadatas( - host_span[const_FileMetaData_ptr]( - metadata_ptrs.data(), - metadata_ptrs.size(), - ) - ) + c_metadatas.reserve(metadata_ptrs.size()) + for i in range(metadata_ptrs.size()): + c_metadatas.push_back(dereference(metadata_ptrs[i])) c_result = move( cpp_read_parquet( move(sources), diff --git a/python/pylibcudf/pylibcudf/libcudf/io/parquet.pxd b/python/pylibcudf/pylibcudf/libcudf/io/parquet.pxd index 184c532b2d46..0f5a531dea8a 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/parquet.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/parquet.pxd @@ -351,26 +351,3 @@ cdef extern from "cudf/io/parquet.hpp" namespace "cudf::io" nogil: cdef unique_ptr[vector[uint8_t]] merge_row_group_metadata( const vector[unique_ptr[vector[uint8_t]]]& metadata_list ) except +libcudf_exception_handler - -from pylibcudf.libcudf.utilities.span cimport host_span - -ctypedef const FileMetaData* const_FileMetaData_ptr - -cdef extern from *: - """ - #include - #include - #include - - std::vector copy_parquet_metadatas( - cudf::host_span sources) - { - std::vector result; - result.reserve(sources.size()); - for (auto const* src : sources) { result.push_back(*src); } - return result; - } - """ - cdef vector[FileMetaData] copy_parquet_metadatas( - host_span[const_FileMetaData_ptr] sources - ) except +libcudf_exception_handler nogil From 420469a0e6ca7c5e0c173e945510ea3dd456643a Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Thu, 13 Aug 2026 14:40:08 +0000 Subject: [PATCH 24/26] more doc strings fixes --- python/cudf_polars/cudf_polars/utils/config.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/python/cudf_polars/cudf_polars/utils/config.py b/python/cudf_polars/cudf_polars/utils/config.py index e4d4461687d2..b7a5c85eb03d 100644 --- a/python/cudf_polars/cudf_polars/utils/config.py +++ b/python/cudf_polars/cudf_polars/utils/config.py @@ -67,12 +67,14 @@ class Unspecified: """ - Sentinel value meaning "fall back to environment variable, then built-in default". + Sentinel value meaning "no value was explicitly provided". The singleton instance :data:`UNSPECIFIED` is used as the default for every - :class:`StreamingOptions` field. When a field is still ``UNSPECIFIED`` after - construction (i.e. neither an explicit value nor an environment variable was provided), - the underlying library applies its own built-in default. + :class:`StreamingOptions` field, as well as for + :attr:`ParquetOptions.prefetch_file_metadata`. When a field is still + ``UNSPECIFIED`` after construction (i.e. neither an explicit value nor a + matching environment variable was provided), the consuming component decides + on the semantics. """ _instance: Unspecified | None = None @@ -89,11 +91,12 @@ def __repr__(self) -> str: UNSPECIFIED = Unspecified() -"""Singleton sentinel for all :class:`StreamingOptions` fields. +"""Singleton sentinel for all :class:`StreamingOptions` fields, as well as for +:attr:`ParquetOptions.prefetch_file_metadata`. A field set to ``UNSPECIFIED`` after construction means no explicit value and no -matching environment variable was found; the underlying library will apply its own -built-in default. +matching environment variable was found; the consuming component decides on the +semantics. """ From 813740fec44cc563f68f3f589aaa63baffd56a8a Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Thu, 13 Aug 2026 14:45:08 +0000 Subject: [PATCH 25/26] trigger github update From d4ecacec0cbdcb420a786416ff3b9e50111f0ca9 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Sun, 16 Aug 2026 23:47:30 +0000 Subject: [PATCH 26/26] docs --- python/cudf_polars/cudf_polars/utils/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cudf_polars/cudf_polars/utils/config.py b/python/cudf_polars/cudf_polars/utils/config.py index b7a5c85eb03d..ec144fc2fc73 100644 --- a/python/cudf_polars/cudf_polars/utils/config.py +++ b/python/cudf_polars/cudf_polars/utils/config.py @@ -71,7 +71,7 @@ class Unspecified: The singleton instance :data:`UNSPECIFIED` is used as the default for every :class:`StreamingOptions` field, as well as for - :attr:`ParquetOptions.prefetch_file_metadata`. When a field is still + ``ParquetOptions.prefetch_file_metadata``. When a field is still ``UNSPECIFIED`` after construction (i.e. neither an explicit value nor a matching environment variable was provided), the consuming component decides on the semantics. @@ -92,7 +92,7 @@ def __repr__(self) -> str: UNSPECIFIED = Unspecified() """Singleton sentinel for all :class:`StreamingOptions` fields, as well as for -:attr:`ParquetOptions.prefetch_file_metadata`. +``ParquetOptions.prefetch_file_metadata``. A field set to ``UNSPECIFIED`` after construction means no explicit value and no matching environment variable was found; the consuming component decides on the