diff --git a/Cargo.lock b/Cargo.lock index 06c12a9e0757f..c81da77116ed5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1745,6 +1745,7 @@ dependencies = [ "criterion", "ctor", "dashmap", + "datafusion", "datafusion-catalog", "datafusion-catalog-listing", "datafusion-common", @@ -1927,6 +1928,7 @@ dependencies = [ "arrow-schema", "chrono", "criterion", + "datafusion-common", "foldhash 0.2.0", "half", "hashbrown 0.17.1", @@ -1969,6 +1971,7 @@ dependencies = [ "criterion", "datafusion-common", "datafusion-common-runtime", + "datafusion-datasource", "datafusion-execution", "datafusion-expr", "datafusion-functions", @@ -2206,6 +2209,7 @@ dependencies = [ "ctor", "datafusion-common", "datafusion-doc", + "datafusion-expr", "datafusion-expr-common", "datafusion-functions-aggregate-common", "datafusion-functions-window-common", @@ -2545,6 +2549,7 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", + "datafusion-physical-plan", "datafusion-proto-common", "datafusion-proto-models", "futures", diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index aa508123d6e05..4e3ec4e0143b0 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -75,6 +75,7 @@ workspace = true [dev-dependencies] ctor = { workspace = true } +datafusion-common = { workspace = true, features = ["test_utils"] } insta = { workspace = true } insta-cmd = "0.7.0" rstest = { workspace = true } diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml index 6d6d917ac46ec..84ebe0f6f5c44 100644 --- a/datafusion-examples/Cargo.toml +++ b/datafusion-examples/Cargo.toml @@ -51,6 +51,8 @@ bytes = { workspace = true } dashmap = { workspace = true } # note only use main datafusion crate for examples base64 = "0.23.0" +datafusion = { workspace = true, features = ["test_utils"] } +datafusion-common = { workspace = true, features = ["test_utils"] } datafusion-expr = { workspace = true } datafusion-physical-expr-adapter = { workspace = true } datafusion-proto = { workspace = true, features = ["parquet"] } diff --git a/datafusion/catalog/Cargo.toml b/datafusion/catalog/Cargo.toml index 1009e9aee477b..e95abf6db518b 100644 --- a/datafusion/catalog/Cargo.toml +++ b/datafusion/catalog/Cargo.toml @@ -30,6 +30,10 @@ version.workspace = true [package.metadata.docs.rs] all-features = true +[features] +# Exposes internal test utilities for use by other DataFusion crates. +test_utils = [] + [dependencies] arrow = { workspace = true } async-trait = { workspace = true } diff --git a/datafusion/catalog/src/memory/table.rs b/datafusion/catalog/src/memory/table.rs index 9cfa32e0d09fe..4d1413f5fc6c4 100644 --- a/datafusion/catalog/src/memory/table.rs +++ b/datafusion/catalog/src/memory/table.rs @@ -71,8 +71,17 @@ pub use datafusion_datasource::memory::PartitionData; #[derive(Debug)] pub struct MemTable { schema: SchemaRef, - // batches used to be pub(crate), but it's needed to be public for the tests + /// # Public Only for Internal Use: + /// Batches exposed for the memory table tests in `datafusion`. Not part of the + /// supported public API. + /// See the [API health policy] for details. + /// + /// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis + #[cfg(any(test, feature = "test_utils"))] + #[doc(hidden)] pub batches: Vec, + #[cfg(not(any(test, feature = "test_utils")))] + batches: Vec, constraints: Constraints, column_defaults: HashMap, /// Optional pre-known sort order(s). Must be `SortExpr`s. diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml index 9ee199fe82f28..63f5402a3c2f0 100644 --- a/datafusion/common/Cargo.toml +++ b/datafusion/common/Cargo.toml @@ -41,6 +41,8 @@ workspace = true name = "datafusion_common" [features] +# Exposes internal test utilities for use by other DataFusion crates. +test_utils = [] backtrace = [] parquet_encryption = [ "parquet", @@ -95,6 +97,7 @@ web-time = "1.1.0" [dev-dependencies] chrono = { workspace = true } criterion = { workspace = true } +datafusion-common = { workspace = true, features = ["test_utils"] } insta = { workspace = true } rand = { workspace = true } sqlparser = { workspace = true } diff --git a/datafusion/common/src/lib.rs b/datafusion/common/src/lib.rs index 2eebfe4963057..127eb60137741 100644 --- a/datafusion/common/src/lib.rs +++ b/datafusion/common/src/lib.rs @@ -60,6 +60,13 @@ pub mod rounding; pub mod scalar; pub mod spans; pub mod stats; +/// # Public Only for Internal Use: +/// Shared test helpers for DataFusion crates. Not part of the supported public API. +/// See the [API health policy] for details. +/// +/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis +#[cfg(any(test, feature = "test_utils"))] +#[doc(hidden)] pub mod test_util; pub mod tree_node; pub mod types; diff --git a/datafusion/common/src/test_util.rs b/datafusion/common/src/test_util.rs index 348fe2ef547f0..89dad5f67d441 100644 --- a/datafusion/common/src/test_util.rs +++ b/datafusion/common/src/test_util.rs @@ -68,7 +68,14 @@ pub fn format_batches(results: &[RecordBatch]) -> Result { let expected_lines: Vec = @@ -117,7 +124,14 @@ pub fn batches_to_sort_string(batches: &[RecordBatch]) -> String { /// Expects to be called about like this: /// /// `assert_batch_sorted_eq!(expected_lines: &[&str], batches: &[RecordBatch])` +/// +/// # Public Only for Internal Use: +/// Shared test assertions for DataFusion crates. Not part of the supported public API. +/// See the [API health policy] for details. +/// +/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis #[macro_export] +#[doc(hidden)] macro_rules! assert_batches_sorted_eq { ($EXPECTED_LINES: expr, $CHUNKS: expr) => { let mut expected_lines: Vec = @@ -159,7 +173,14 @@ macro_rules! assert_batches_sorted_eq { /// messages are on the same line as the failure; /// /// Both arguments must be convertible into Strings ([`Into`]<[`String`]>) +/// +/// # Public Only for Internal Use: +/// Shared test assertions for DataFusion crates. Not part of the supported public API. +/// See the [API health policy] for details. +/// +/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis #[macro_export] +#[doc(hidden)] macro_rules! assert_contains { ($ACTUAL: expr, $EXPECTED: expr) => { let actual_value: String = $ACTUAL.into(); @@ -182,7 +203,14 @@ macro_rules! assert_contains { /// messages are on the same line as the failure; /// /// Both arguments must be convertible into Strings ([`Into`]<[`String`]>) +/// +/// # Public Only for Internal Use: +/// Shared test assertions for DataFusion crates. Not part of the supported public API. +/// See the [API health policy] for details. +/// +/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis #[macro_export] +#[doc(hidden)] macro_rules! assert_not_contains { ($ACTUAL: expr, $UNEXPECTED: expr) => { let actual_value: String = $ACTUAL.into(); @@ -318,7 +346,14 @@ pub fn get_data_dir( } } +/// # Public Only for Internal Use: +/// Array construction for test macros in DataFusion crates. Not part of the supported +/// public API. +/// See the [API health policy] for details. +/// +/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis #[macro_export] +#[doc(hidden)] macro_rules! create_array { (Boolean, $values: expr) => { std::sync::Arc::new($crate::arrow::array::BooleanArray::from($values)) @@ -377,8 +412,16 @@ macro_rules! create_array { /// ("c", Utf8, vec!["alpha", "beta", "gamma"]) /// ); /// ``` +/// +/// # Public Only for Internal Use: +/// Record batch construction for tests in DataFusion crates. Not part of the supported +/// public API. +/// See the [API health policy] for details. +/// +/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis #[deprecated(since = "55.0.0", note = "Use `arrow::array::record_batch` instead")] #[macro_export] +#[doc(hidden)] macro_rules! record_batch { ($(($name: expr, $type: ident, $values: expr)),*) => { { diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index 222c0ec688b78..7eb7f0020cc99 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -39,6 +39,15 @@ all-features = true workspace = true [features] +# Exposes internal test utilities for use by other DataFusion crates. +test_utils = [ + "datafusion-catalog/test_utils", + "datafusion-common/test_utils", + "datafusion-datasource/test_utils", + "datafusion-expr/test_utils", + "datafusion-physical-expr/test_utils", + "datafusion-physical-plan/test_utils", +] # Enables the `print_config_docs`, `print_runtime_config_docs` and # `print_functions_docs` binaries, which regenerate the docs under # `docs/source/user-guide`. Off by default: they are only run by @@ -171,6 +180,7 @@ async-trait = { workspace = true } criterion = { workspace = true, features = ["async_tokio", "async_futures"] } ctor = { workspace = true } dashmap = "6.2.1" +datafusion = { workspace = true, features = ["test_utils"] } datafusion-doc = { workspace = true } datafusion-functions-window-common = { workspace = true } datafusion-macros = { workspace = true } diff --git a/datafusion/core/src/lib.rs b/datafusion/core/src/lib.rs index 28ea4b4490c3e..78c084ce482c1 100644 --- a/datafusion/core/src/lib.rs +++ b/datafusion/core/src/lib.rs @@ -854,8 +854,23 @@ pub mod physical_plan { pub use datafusion_physical_plan::*; } -// Reexport testing macros for compatibility +/// # Public Only for Internal Use: +/// Test assertions re-exported for compatibility across DataFusion crates. Not part of +/// the supported public API. +/// See the [API health policy] for details. +/// +/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis +#[cfg(any(test, feature = "test_utils"))] +#[doc(hidden)] pub use datafusion_common::assert_batches_eq; +/// # Public Only for Internal Use: +/// Test assertions re-exported for compatibility across DataFusion crates. Not part of +/// the supported public API. +/// See the [API health policy] for details. +/// +/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis +#[cfg(any(test, feature = "test_utils"))] +#[doc(hidden)] pub use datafusion_common::assert_batches_sorted_eq; /// re-export of [`datafusion_sql`] crate @@ -895,10 +910,24 @@ pub mod variable { pub use datafusion_expr::var_provider::{VarProvider, VarType}; } +/// # Public Only for Internal Use: +/// Shared test helpers for DataFusion crates. Not part of the supported public API. +/// See the [API health policy] for details. +/// +/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis #[cfg(not(target_arch = "wasm32"))] +#[cfg(any(test, feature = "test_utils"))] +#[doc(hidden)] pub mod test; mod schema_equivalence; +/// # Public Only for Internal Use: +/// Shared test helpers for DataFusion crates. Not part of the supported public API. +/// See the [API health policy] for details. +/// +/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis +#[cfg(any(test, feature = "test_utils"))] +#[doc(hidden)] pub mod test_util; #[cfg(doctest)] diff --git a/datafusion/datasource-avro/Cargo.toml b/datafusion/datasource-avro/Cargo.toml index 63e949e21def9..671de1d77df90 100644 --- a/datafusion/datasource-avro/Cargo.toml +++ b/datafusion/datasource-avro/Cargo.toml @@ -59,6 +59,7 @@ object_store = { workspace = true } tokio = { workspace = true } [dev-dependencies] +datafusion-common = { workspace = true, features = ["test_utils"] } # Note: add additional linter rules in lib.rs. # Rust does not support workspace + new linter rules in subcrates yet diff --git a/datafusion/datasource-parquet/Cargo.toml b/datafusion/datasource-parquet/Cargo.toml index a2589af19a6ee..be0e404b07705 100644 --- a/datafusion/datasource-parquet/Cargo.toml +++ b/datafusion/datasource-parquet/Cargo.toml @@ -60,6 +60,7 @@ tokio = { workspace = true } [dev-dependencies] chrono = { workspace = true } criterion = { workspace = true } +datafusion-common = { workspace = true, features = ["test_utils"] } datafusion-functions = { workspace = true } datafusion-functions-nested = { workspace = true } tempfile = { workspace = true } diff --git a/datafusion/datasource/Cargo.toml b/datafusion/datasource/Cargo.toml index f09447b694f52..d06a42093a1d8 100644 --- a/datafusion/datasource/Cargo.toml +++ b/datafusion/datasource/Cargo.toml @@ -31,6 +31,8 @@ version.workspace = true all-features = true [features] +# Exposes internal test utilities for use by other DataFusion crates. +test_utils = [] backtrace = ["datafusion-common/backtrace"] compression = ["async-compression", "liblzma", "bzip2", "flate2", "zstd", "tokio-util"] default = ["compression"] @@ -83,6 +85,8 @@ zstd = { workspace = true, optional = true } [dev-dependencies] criterion = { workspace = true } +datafusion-common = { workspace = true, features = ["test_utils"] } +datafusion-datasource = { workspace = true, features = ["test_utils"] } datafusion-functions = { workspace = true } insta = { workspace = true } tempfile = { workspace = true } diff --git a/datafusion/datasource/src/mod.rs b/datafusion/datasource/src/mod.rs index d0bf64d81c71f..df5951d4d4909 100644 --- a/datafusion/datasource/src/mod.rs +++ b/datafusion/datasource/src/mod.rs @@ -58,10 +58,13 @@ pub mod url; pub mod write; pub use self::file::as_file_source; pub use self::url::ListingTableUrl; +#[cfg(any(test, feature = "test_utils"))] use crate::file_groups::FileGroup; use arrow::datatypes::SchemaRef; use chrono::TimeZone; -use datafusion_common::stats::{Precision, is_known_empty}; +use datafusion_common::stats::Precision; +#[cfg(any(test, feature = "test_utils"))] +use datafusion_common::stats::is_known_empty; use datafusion_common::{ColumnStatistics, Result, TableReference}; use datafusion_common::{ScalarValue, Statistics}; use datafusion_physical_expr::LexOrdering; @@ -450,6 +453,14 @@ impl From for PartitionedFile { /// File 2: [40, 140] /// File 3: [60, 160] /// File 4: [80, 180] +/// +/// # Public Only for Internal Use: +/// Used by file group tests and benchmarks. Not part of the supported public API. +/// See the [API health policy] for details. +/// +/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis +#[cfg(any(test, feature = "test_utils"))] +#[doc(hidden)] pub fn generate_test_files(num_files: usize, overlap_factor: f64) -> Vec { let mut files = Vec::with_capacity(num_files); if num_files == 0 { @@ -500,8 +511,15 @@ pub fn generate_test_files(num_files: usize, overlap_factor: f64) -> Vec bool { for group in file_groups { // Known-empty files contribute no rows and may not have min/max diff --git a/datafusion/expr-common/Cargo.toml b/datafusion/expr-common/Cargo.toml index 026c1e0bc316b..acb439b372935 100644 --- a/datafusion/expr-common/Cargo.toml +++ b/datafusion/expr-common/Cargo.toml @@ -47,5 +47,6 @@ indexmap = { workspace = true } itertools = { workspace = true } [dev-dependencies] +datafusion-common = { workspace = true, features = ["test_utils"] } insta = { workspace = true } rstest = { workspace = true } diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml index 74b85501dde91..5c9e5cf426211 100644 --- a/datafusion/expr/Cargo.toml +++ b/datafusion/expr/Cargo.toml @@ -41,6 +41,8 @@ workspace = true name = "datafusion_expr" [features] +# Exposes internal test utilities for use by other DataFusion crates. +test_utils = [] default = ["sql"] # Enables protobuf conversions for the expression types owned by this crate. # Off by default so consumers that never serialize plans pay nothing. Mirrors @@ -71,6 +73,8 @@ sqlparser = { workspace = true, optional = true } [dev-dependencies] criterion = { workspace = true } ctor = { workspace = true } +datafusion-common = { workspace = true, features = ["test_utils"] } +datafusion-expr = { workspace = true, features = ["test_utils"] } env_logger = { workspace = true } insta = { workspace = true } # Makes sure `test_display_pg_json` behaves in a consistent way regardless of diff --git a/datafusion/expr/src/lib.rs b/datafusion/expr/src/lib.rs index ed9b4df4ce2bd..cffd3bb230b5b 100644 --- a/datafusion/expr/src/lib.rs +++ b/datafusion/expr/src/lib.rs @@ -88,6 +88,14 @@ pub mod preimage; pub mod ptr_eq; #[cfg(not(feature = "sql"))] pub mod sql; +/// # Public Only for Internal Use: +/// Function stubs shared by tests in DataFusion crates. Not part of the supported +/// public API. +/// See the [API health policy] for details. +/// +/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis +#[cfg(any(test, feature = "test_utils"))] +#[doc(hidden)] pub mod test; pub mod tree_node; pub mod type_coercion; diff --git a/datafusion/ffi/Cargo.toml b/datafusion/ffi/Cargo.toml index affcff3dbdcd9..9e9dcbe9e54d0 100644 --- a/datafusion/ffi/Cargo.toml +++ b/datafusion/ffi/Cargo.toml @@ -75,7 +75,8 @@ stabby = "72.1.2" tokio = { workspace = true } [dev-dependencies] -datafusion = { workspace = true, default-features = false, features = ["sql"] } +datafusion = { workspace = true, default-features = false, features = ["test_utils", "sql"] } +datafusion-common = { workspace = true, features = ["test_utils"] } datafusion-functions = { workspace = true } datafusion-functions-aggregate = { workspace = true } datafusion-functions-aggregate-common = { workspace = true } diff --git a/datafusion/functions-aggregate-common/Cargo.toml b/datafusion/functions-aggregate-common/Cargo.toml index 463034d4d6de3..977952845400f 100644 --- a/datafusion/functions-aggregate-common/Cargo.toml +++ b/datafusion/functions-aggregate-common/Cargo.toml @@ -41,6 +41,8 @@ workspace = true name = "datafusion_functions_aggregate_common" [features] +# Exposes internal test utilities for use by other DataFusion crates. +test_utils = [] # Used for testing ONLY: causes all values to hash to the same value (test for collisions) force_hash_collisions = ["datafusion-common/force_hash_collisions"] diff --git a/datafusion/functions-aggregate-common/src/tdigest.rs b/datafusion/functions-aggregate-common/src/tdigest.rs index 860f56f6c00dc..6adb2e9d92dd8 100644 --- a/datafusion/functions-aggregate-common/src/tdigest.rs +++ b/datafusion/functions-aggregate-common/src/tdigest.rs @@ -206,7 +206,16 @@ impl TDigest { v.clamp(min, max) } - // public for testing in other modules + /// Merge unsorted values. + /// + /// # Public Only for Internal Use: + /// Used by tests in `datafusion-functions-aggregate`. Not part of the supported + /// public API. + /// See the [API health policy] for details. + /// + /// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis + #[cfg(any(test, feature = "test_utils"))] + #[doc(hidden)] pub fn merge_unsorted_f64(&self, unsorted_values: Vec) -> TDigest { let mut values = unsorted_values; values.sort_by(|a, b| a.total_cmp(b)); diff --git a/datafusion/functions-aggregate/Cargo.toml b/datafusion/functions-aggregate/Cargo.toml index ead59720d216e..05145dcb3928d 100644 --- a/datafusion/functions-aggregate/Cargo.toml +++ b/datafusion/functions-aggregate/Cargo.toml @@ -60,6 +60,7 @@ num-traits = { workspace = true } [dev-dependencies] arrow = { workspace = true, features = ["test_utils"] } criterion = { workspace = true } +datafusion-functions-aggregate-common = { workspace = true, features = ["test_utils"] } rand = { workspace = true } [[bench]] diff --git a/datafusion/functions-nested/Cargo.toml b/datafusion/functions-nested/Cargo.toml index 728255cecf252..745e27b4bb39d 100644 --- a/datafusion/functions-nested/Cargo.toml +++ b/datafusion/functions-nested/Cargo.toml @@ -66,6 +66,7 @@ memchr = { workspace = true } [dev-dependencies] arrow = { workspace = true, features = ["test_utils"] } criterion = { workspace = true, features = ["async_tokio"] } +datafusion-common = { workspace = true, features = ["test_utils"] } rand = { workspace = true } # used to test array_transform datafusion-physical-expr = { workspace = true } diff --git a/datafusion/functions/Cargo.toml b/datafusion/functions/Cargo.toml index 28765bdda9e2e..5e9cea6a61ed5 100644 --- a/datafusion/functions/Cargo.toml +++ b/datafusion/functions/Cargo.toml @@ -94,6 +94,7 @@ uuid = { workspace = true, features = ["v4"], optional = true } arrow = { workspace = true, features = ["test_utils"] } criterion = { workspace = true } ctor = { workspace = true } +datafusion-common = { workspace = true, features = ["test_utils"] } env_logger = { workspace = true } rand = { workspace = true } tokio = { workspace = true, features = ["macros", "rt", "sync"] } diff --git a/datafusion/optimizer/Cargo.toml b/datafusion/optimizer/Cargo.toml index 31a73062e119e..49d9a7f116411 100644 --- a/datafusion/optimizer/Cargo.toml +++ b/datafusion/optimizer/Cargo.toml @@ -69,6 +69,8 @@ regex-syntax = "0.8.9" async-trait = { workspace = true } criterion = { workspace = true } ctor = { workspace = true } +datafusion-common = { workspace = true, features = ["test_utils"] } +datafusion-expr = { workspace = true, features = ["test_utils"] } datafusion-functions-aggregate = { workspace = true } datafusion-functions-window = { workspace = true } datafusion-functions-window-common = { workspace = true } diff --git a/datafusion/physical-expr-adapter/Cargo.toml b/datafusion/physical-expr-adapter/Cargo.toml index 453c8bdaacb4a..af97f87def834 100644 --- a/datafusion/physical-expr-adapter/Cargo.toml +++ b/datafusion/physical-expr-adapter/Cargo.toml @@ -28,3 +28,4 @@ itertools = { workspace = true } workspace = true [dev-dependencies] +datafusion-common = { workspace = true, features = ["test_utils"] } diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.toml index 858dd86b7790d..1d543e3733163 100644 --- a/datafusion/physical-expr/Cargo.toml +++ b/datafusion/physical-expr/Cargo.toml @@ -41,6 +41,8 @@ workspace = true name = "datafusion_physical_expr" [features] +# Exposes internal test utilities for use by other DataFusion crates. +test_utils = [] recursive_protection = ["dep:recursive"] # Forwards the `proto` feature to `datafusion-physical-expr-common`, exposing # `PhysicalExpr::to_proto` and letting expressions in this crate implement it. @@ -71,6 +73,8 @@ half = { workspace = true } [dev-dependencies] arrow = { workspace = true, features = ["test_utils"] } criterion = { workspace = true } +datafusion-common = { workspace = true, features = ["test_utils"] } +datafusion-expr = { workspace = true, features = ["test_utils"] } datafusion-functions = { workspace = true } insta = { workspace = true } rand = { workspace = true } diff --git a/datafusion/physical-expr/src/intervals/mod.rs b/datafusion/physical-expr/src/intervals/mod.rs index 9752ca27b5a38..452213a1a3ccf 100644 --- a/datafusion/physical-expr/src/intervals/mod.rs +++ b/datafusion/physical-expr/src/intervals/mod.rs @@ -18,5 +18,13 @@ //! Interval arithmetic and constraint propagation library pub mod cp_solver; +/// # Public Only for Internal Use: +/// Interval arithmetic helpers shared by tests in DataFusion crates. Not part of the +/// supported public API. +/// See the [API health policy] for details. +/// +/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis +#[cfg(any(test, feature = "test_utils"))] +#[doc(hidden)] pub mod test_utils; pub mod utils; diff --git a/datafusion/physical-optimizer/Cargo.toml b/datafusion/physical-optimizer/Cargo.toml index cb03303ac3c3f..f6389e4ea260e 100644 --- a/datafusion/physical-optimizer/Cargo.toml +++ b/datafusion/physical-optimizer/Cargo.toml @@ -58,5 +58,6 @@ recursive = { workspace = true, optional = true } datafusion-expr = { workspace = true } datafusion-functions = { workspace = true } datafusion-functions-window = { workspace = true } +datafusion-physical-plan = { workspace = true, features = ["test_utils"] } insta = { workspace = true } tokio = { workspace = true } diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.toml index 6a717defb95fa..4a37b15ccaea6 100644 --- a/datafusion/physical-plan/Cargo.toml +++ b/datafusion/physical-plan/Cargo.toml @@ -39,6 +39,7 @@ workspace = true [features] force_hash_collisions = [] +# Exposes internal test utilities for use by other DataFusion crates. test_utils = ["arrow/test_utils"] tokio_coop = [] tokio_coop_fallback = [] @@ -93,8 +94,12 @@ tokio = { workspace = true, features = ["time"] } [dev-dependencies] arrow-data = { workspace = true } criterion = { workspace = true, features = ["async_futures"] } +datafusion-common = { workspace = true, features = ["test_utils"] } +datafusion-expr = { workspace = true, features = ["test_utils"] } datafusion-functions-aggregate = { workspace = true } datafusion-functions-window = { workspace = true } +datafusion-physical-expr = { workspace = true, features = ["test_utils"] } +datafusion-physical-plan = { workspace = true, features = ["test_utils"] } insta = { workspace = true } rand = { workspace = true } rstest = { workspace = true } diff --git a/datafusion/physical-plan/src/distribution_requirements.rs b/datafusion/physical-plan/src/distribution_requirements.rs index f7a17ddc7b8d6..17370789e34f4 100644 --- a/datafusion/physical-plan/src/distribution_requirements.rs +++ b/datafusion/physical-plan/src/distribution_requirements.rs @@ -188,6 +188,13 @@ impl InputDistributionRequirements { /// Independent per-child requirements are intentionally ignored here, use /// [`Self::child_satisfaction`] for those checks. An empty result means all /// co-partitioning requirements are satisfied. + /// + /// # Public Only for Internal Use: + /// `datafusion-physical-optimizer` uses this to enforce distribution + /// requirements. It is not part of the supported public API. + /// See the [API health policy] for details. + /// + /// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis #[doc(hidden)] pub fn unsatisfied_co_partitioned_children( &self, diff --git a/datafusion/physical-plan/src/joins/hash_join/mod.rs b/datafusion/physical-plan/src/joins/hash_join/mod.rs index 7c1b0d76c2f73..198e0789b196b 100644 --- a/datafusion/physical-plan/src/joins/hash_join/mod.rs +++ b/datafusion/physical-plan/src/joins/hash_join/mod.rs @@ -18,7 +18,16 @@ //! [`HashJoinExec`] Partitioned Hash Join Operator pub use exec::{HashJoinExec, HashJoinExecBuilder}; -pub use partitioned_hash_eval::{HashExpr, HashTableLookupExpr, SeededRandomState}; +/// # Public Only for Internal Use: +/// Exposed for expression serialization tests in `datafusion-proto`. Not part of the +/// supported public API. +/// See the [API health policy] for details. +/// +/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis +#[cfg(any(test, feature = "test_utils"))] +#[doc(hidden)] +pub use partitioned_hash_eval::HashTableLookupExpr; +pub use partitioned_hash_eval::{HashExpr, SeededRandomState}; mod exec; mod inlist_builder; diff --git a/datafusion/physical-plan/src/joins/mod.rs b/datafusion/physical-plan/src/joins/mod.rs index 10e5793e0ff32..b0ff814b87821 100644 --- a/datafusion/physical-plan/src/joins/mod.rs +++ b/datafusion/physical-plan/src/joins/mod.rs @@ -24,9 +24,16 @@ use arrow::array::BooleanBufferBuilder; pub use asof_join::{AsOfJoinExec, AsOfMatchExpr}; pub use cross_join::CrossJoinExec; use datafusion_physical_expr::PhysicalExprRef; -pub use hash_join::{ - HashExpr, HashJoinExec, HashJoinExecBuilder, HashTableLookupExpr, SeededRandomState, -}; +/// # Public Only for Internal Use: +/// Exposed for expression serialization tests in `datafusion-proto`. Not part of the +/// supported public API. +/// See the [API health policy] for details. +/// +/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis +#[cfg(any(test, feature = "test_utils"))] +#[doc(hidden)] +pub use hash_join::HashTableLookupExpr; +pub use hash_join::{HashExpr, HashJoinExec, HashJoinExecBuilder, SeededRandomState}; pub use nested_loop_join::{NestedLoopJoinExec, NestedLoopJoinExecBuilder}; use parking_lot::Mutex; // Note: SortMergeJoin is not used in plans yet @@ -51,9 +58,17 @@ mod array_map; mod join_filter; /// Hash map implementations for join operations. /// -/// Note: This module is public for internal testing purposes only -/// and is not guaranteed to be stable across versions. +/// # Public Only for Internal Use: +/// Exposed for hash join tests in `datafusion-proto`. Not part of the supported public +/// API. +/// See the [API health policy] for details. +/// +/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis +#[cfg(any(test, feature = "test_utils"))] +#[doc(hidden)] pub mod join_hash_map; +#[cfg(not(any(test, feature = "test_utils")))] +mod join_hash_map; use array_map::ArrayMap; use utils::JoinHashMapType; diff --git a/datafusion/physical-plan/src/lib.rs b/datafusion/physical-plan/src/lib.rs index 5ff6cec374ee1..620648696a6fd 100644 --- a/datafusion/physical-plan/src/lib.rs +++ b/datafusion/physical-plan/src/lib.rs @@ -115,4 +115,11 @@ pub mod udaf { pub use datafusion_physical_expr::aggregate::AggregateFunctionExpr; } +/// # Public Only for Internal Use: +/// Shared test helpers for DataFusion crates. Not part of the supported public API. +/// See the [API health policy] for details. +/// +/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis +#[cfg(any(test, feature = "test_utils"))] +#[doc(hidden)] pub mod test; diff --git a/datafusion/physical-plan/src/spill/mod.rs b/datafusion/physical-plan/src/spill/mod.rs index 3f56d2a6e0aaa..465e71b68de11 100644 --- a/datafusion/physical-plan/src/spill/mod.rs +++ b/datafusion/physical-plan/src/spill/mod.rs @@ -24,7 +24,13 @@ pub mod spill_pool; use datafusion_execution::spill_file::{AsyncSpillWriter, SpillWriter}; // Moved for refactor, re-export to keep the public API stable pub use datafusion_common::utils::memory::get_record_batch_memory_size; -// Re-export SpillManager for doctests only (hidden from public docs) +/// # Public Only for Internal Use: +/// Exposed here for doctests and the spilling tests in `datafusion`. Not part of the +/// supported public API. +/// See the [API health policy] for details. +/// +/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis +#[cfg(any(test, feature = "test_utils"))] #[doc(hidden)] pub use spill_manager::SpillManager; diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml index 4b11bea01103f..49bcde91c42bd 100644 --- a/datafusion/proto/Cargo.toml +++ b/datafusion/proto/Cargo.toml @@ -86,11 +86,14 @@ datafusion = { workspace = true, default-features = false, features = [ "compression", "datetime_expressions", "nested_expressions", + "test_utils", "unicode_expressions", ] } +datafusion-common = { workspace = true, features = ["test_utils"] } datafusion-functions = { workspace = true, default-features = true } datafusion-functions-aggregate = { workspace = true } datafusion-functions-window-common = { workspace = true } +datafusion-physical-plan = { workspace = true, features = ["test_utils"] } doc-comment = { workspace = true } flate2 = { workspace = true } pretty_assertions = "1.4" diff --git a/datafusion/pruning/Cargo.toml b/datafusion/pruning/Cargo.toml index a914a0a079bcd..dcb557d750cf4 100644 --- a/datafusion/pruning/Cargo.toml +++ b/datafusion/pruning/Cargo.toml @@ -27,6 +27,7 @@ log = { workspace = true } [dev-dependencies] criterion = { workspace = true } +datafusion-common = { workspace = true, features = ["test_utils"] } datafusion-expr = { workspace = true } datafusion-functions-nested = { workspace = true } insta = { workspace = true } diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.toml index 08309a931d96b..a931deedcddcc 100644 --- a/datafusion/spark/Cargo.toml +++ b/datafusion/spark/Cargo.toml @@ -73,6 +73,7 @@ arrow = { workspace = true, features = ["test_utils"] } criterion = { workspace = true } # for SessionStateBuilderSpark tests datafusion = { workspace = true, default-features = false, features = ["sql"] } +datafusion-common = { workspace = true, features = ["test_utils"] } tokio = { workspace = true, features = ["rt"] } [[bench]] diff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.toml index 318f8934639aa..8234371b70e1f 100644 --- a/datafusion/sql/Cargo.toml +++ b/datafusion/sql/Cargo.toml @@ -67,6 +67,8 @@ stacker = { workspace = true, optional = true } [dev-dependencies] ctor = { workspace = true } # please do not move these dependencies to the main dependencies section +datafusion-common = { workspace = true, features = ["test_utils"] } +datafusion-expr = { workspace = true, features = ["test_utils"] } datafusion-functions = { workspace = true, default-features = true } datafusion-functions-aggregate = { workspace = true } datafusion-functions-nested = { workspace = true, features = ["sql"] } diff --git a/datafusion/sqllogictest/Cargo.toml b/datafusion/sqllogictest/Cargo.toml index c43f78f846092..e004065c6abea 100644 --- a/datafusion/sqllogictest/Cargo.toml +++ b/datafusion/sqllogictest/Cargo.toml @@ -46,7 +46,7 @@ bigdecimal = { workspace = true } bytes = { workspace = true, optional = true } chrono = { workspace = true, optional = true } clap = { version = "4.5.60", features = ["derive", "env"] } -datafusion = { workspace = true, default-features = true, features = ["avro"] } +datafusion = { workspace = true, default-features = true, features = ["test_utils", "avro"] } datafusion-spark = { workspace = true, features = ["core"] } datafusion-substrait = { workspace = true, default-features = true, optional = true } futures = { workspace = true } diff --git a/datafusion/substrait/Cargo.toml b/datafusion/substrait/Cargo.toml index a0f203cec8db6..77af9186684fc 100644 --- a/datafusion/substrait/Cargo.toml +++ b/datafusion/substrait/Cargo.toml @@ -49,7 +49,7 @@ url = { workspace = true } tokio = { workspace = true, features = ["fs"] } [dev-dependencies] -datafusion = { workspace = true, features = ["nested_expressions", "unicode_expressions"] } +datafusion = { workspace = true, features = ["test_utils", "nested_expressions", "unicode_expressions"] } datafusion-functions-aggregate = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } diff --git a/datafusion/wasmtest/Cargo.toml b/datafusion/wasmtest/Cargo.toml index d6cea2e68d384..7e509e495c59c 100644 --- a/datafusion/wasmtest/Cargo.toml +++ b/datafusion/wasmtest/Cargo.toml @@ -61,6 +61,7 @@ wasm-bindgen = "0.2.99" [dev-dependencies] bytes = { workspace = true } +datafusion-common = { workspace = true, features = ["test_utils"] } futures = { workspace = true } object_store = { workspace = true } # needs to be compiled diff --git a/docs/source/contributor-guide/api-health.md b/docs/source/contributor-guide/api-health.md index 8737df280f4f8..ad0c6118a6916 100644 --- a/docs/source/contributor-guide/api-health.md +++ b/docs/source/contributor-guide/api-health.md @@ -43,6 +43,10 @@ Examples of non-breaking changes include: - Marking a function as deprecated (`#[deprecated]`) - Adding a new function to a `trait` with a default implementation + + + + ### DataFusion-internal Public APIs Some internal implementation details require `pub` visibility because they are @@ -51,24 +55,58 @@ extension points. They are hidden from generated documentation and are not part of DataFusion's supported public API, so they may be changed or removed without notice or a deprecation period. Examples include: -1. Test helpers. -2. Operator APIs required by the optimizer to inspect or rewrite execution plans - across crate boundaries. +1. Test/micro-benchmark helpers. +2. Internal utilities, such as operator APIs required by the optimizer to inspect + or rewrite execution plans across crate boundaries. + +Note: Exposing internal APIs for tests or microbenchmarks is not recommended, +though some legacy code does so. For new tests and benchmarks, prefer exercising +observable behavior to simplify maintenance. + +#### Test Helpers + +Hide shared test helpers with `#[doc(hidden)]` and gate them behind the non-default +`test_utils` feature, declared as `test_utils = []` in `[features]`. Unit tests use +`cfg(test)`; tests and benchmarks that link the crate enable `test_utils` in +`[dev-dependencies]`. + +```txt +/// # Public Only for Internal Use: +/// Shared test helpers for DataFusion crates. Not part of the supported public API. +/// See the [API health policy] for details. +/// +/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis +#[cfg(any(test, feature = "test_utils"))] +#[doc(hidden)] +pub mod test; +``` + +```toml +[dev-dependencies] +datafusion-physical-plan = { workspace = true, features = ["test_utils"] } +``` + +#### Internal Utilities + +Utilities needed across crates during normal execution cannot be gated behind +`test_utils`. Add `#[doc(hidden)]` and a doc comment section headed +`# Public Only for Internal Use:`. Name the crate or component that requires access +and explain why the API is not intended for downstream use. -For APIs intended only for internal use, add `#[doc(hidden)]` and a doc comment -section headed `# Public Only for Internal Use:`. Name the crate or component -that requires access and explain why the API is not intended for downstream use. For example: ```txt -impl HashTableLookupExpr { +impl InputDistributionRequirements { /// ... /// /// # Public Only for Internal Use: - /// `datafusion-proto` tests require this constructor, but it is not part of - /// the supported public API. + /// `datafusion-physical-optimizer` uses this to enforce distribution + /// requirements. It is not part of the supported public API. + /// See the [API health policy] for details. + /// + /// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis #[doc(hidden)] - pub fn new(...) {...} + pub fn unsatisfied_co_partitioned_children(...) {...} } ```