chore: Use test_util feature to hide internal public APIs

datafusion-cli/Cargo.toml

@@ -75,6 +75,7 @@ workspace = true
 
 [dev-dependencies]
 ctor = { workspace = true }
+datafusion-common = { workspace = true, features = ["test_utils"] }
 insta = { workspace = true }
 insta-cmd = "0.7.0"
 rstest = { workspace = true }

datafusion-examples/Cargo.toml

@@ -51,6 +51,8 @@ bytes = { workspace = true }
 dashmap = { workspace = true }
 # note only use main datafusion crate for examples
 base64 = "0.23.0"
+datafusion = { workspace = true, features = ["test_utils"] }
+datafusion-common = { workspace = true, features = ["test_utils"] }
 datafusion-expr = { workspace = true }
 datafusion-physical-expr-adapter = { workspace = true }
 datafusion-proto = { workspace = true, features = ["parquet"] }

datafusion/catalog/Cargo.toml

@@ -30,6 +30,10 @@ version.workspace = true
 [package.metadata.docs.rs]
 all-features = true
 
+[features]
+# Exposes internal test utilities for use by other DataFusion crates.
+test_utils = []
+
 [dependencies]
 arrow = { workspace = true }
 async-trait = { workspace = true }

datafusion/catalog/src/memory/table.rs

@@ -71,8 +71,17 @@ pub use datafusion_datasource::memory::PartitionData;
 #[derive(Debug)]
 pub struct MemTable {
     schema: SchemaRef,
-    // batches used to be pub(crate), but it's needed to be public for the tests
+    /// # Public Only for Internal Use:
+    /// Batches exposed for the memory table tests in `datafusion`. Not part of the
+    /// supported public API.
+    /// See the [API health policy] for details.
+    ///
+    /// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis
+    #[cfg(any(test, feature = "test_utils"))]
+    #[doc(hidden)]
     pub batches: Vec<PartitionData>,
+    #[cfg(not(any(test, feature = "test_utils")))]
+    batches: Vec<PartitionData>,
     constraints: Constraints,
     column_defaults: HashMap<String, Expr>,
     /// Optional pre-known sort order(s). Must be `SortExpr`s.

datafusion/common/Cargo.toml

@@ -41,6 +41,8 @@ workspace = true
 name = "datafusion_common"
 
 [features]
+# Exposes internal test utilities for use by other DataFusion crates.
+test_utils = []
 backtrace = []
 parquet_encryption = [
     "parquet",
@@ -95,6 +97,7 @@ web-time = "1.1.0"
 [dev-dependencies]
 chrono = { workspace = true }
 criterion = { workspace = true }
+datafusion-common = { workspace = true, features = ["test_utils"] }
 insta = { workspace = true }
 rand = { workspace = true }
 sqlparser = { workspace = true }

datafusion/common/src/lib.rs

@@ -60,6 +60,13 @@ pub mod rounding;
 pub mod scalar;
 pub mod spans;
 pub mod stats;
+/// # Public Only for Internal Use:
+/// Shared test helpers for DataFusion crates. Not part of the supported public API.
+/// See the [API health policy] for details.
+///
+/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis
+#[cfg(any(test, feature = "test_utils"))]
+#[doc(hidden)]
 pub mod test_util;
 pub mod tree_node;
 pub mod types;

datafusion/common/src/test_util.rs

@@ -68,7 +68,14 @@ pub fn format_batches(results: &[RecordBatch]) -> Result<impl Display, ArrowErro
 /// // compare the formatted output of the record batch with the expected output
 /// assert_batches_eq!(expected, &[batch]);
 /// ```
+///
+/// # Public Only for Internal Use:
+/// Shared test assertions for DataFusion crates. Not part of the supported public API.
+/// See the [API health policy] for details.
+///
+/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis
 #[macro_export]
+#[doc(hidden)]
 macro_rules! assert_batches_eq {
     ($EXPECTED_LINES: expr, $CHUNKS: expr) => {
         let expected_lines: Vec<String> =
@@ -117,7 +124,14 @@ pub fn batches_to_sort_string(batches: &[RecordBatch]) -> String {
 /// Expects to be called about like this:
 ///
 /// `assert_batch_sorted_eq!(expected_lines: &[&str], batches: &[RecordBatch])`
+///
+/// # Public Only for Internal Use:
+/// Shared test assertions for DataFusion crates. Not part of the supported public API.
+/// See the [API health policy] for details.
+///
+/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis
 #[macro_export]
+#[doc(hidden)]
 macro_rules! assert_batches_sorted_eq {
     ($EXPECTED_LINES: expr, $CHUNKS: expr) => {
         let mut expected_lines: Vec<String> =
@@ -159,7 +173,14 @@ macro_rules! assert_batches_sorted_eq {
 /// messages are on the same line as the failure;
 ///
 /// Both arguments must be convertible into Strings ([`Into`]<[`String`]>)
+///
+/// # Public Only for Internal Use:
+/// Shared test assertions for DataFusion crates. Not part of the supported public API.
+/// See the [API health policy] for details.
+///
+/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis
 #[macro_export]
+#[doc(hidden)]
 macro_rules! assert_contains {
     ($ACTUAL: expr, $EXPECTED: expr) => {
         let actual_value: String = $ACTUAL.into();
@@ -182,7 +203,14 @@ macro_rules! assert_contains {
 /// messages are on the same line as the failure;
 ///
 /// Both arguments must be convertible into Strings ([`Into`]<[`String`]>)
+///
+/// # Public Only for Internal Use:
+/// Shared test assertions for DataFusion crates. Not part of the supported public API.
+/// See the [API health policy] for details.
+///
+/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis
 #[macro_export]
+#[doc(hidden)]
 macro_rules! assert_not_contains {
     ($ACTUAL: expr, $UNEXPECTED: expr) => {
         let actual_value: String = $ACTUAL.into();
@@ -318,7 +346,14 @@ pub fn get_data_dir(
     }
 }
 
+/// # Public Only for Internal Use:
+/// Array construction for test macros in DataFusion crates. Not part of the supported
+/// public API.
+/// See the [API health policy] for details.
+///
+/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis
 #[macro_export]
+#[doc(hidden)]
 macro_rules! create_array {
     (Boolean, $values: expr) => {
         std::sync::Arc::new($crate::arrow::array::BooleanArray::from($values))
@@ -377,8 +412,16 @@ macro_rules! create_array {
 ///     ("c", Utf8, vec!["alpha", "beta", "gamma"])
 /// );
 /// ```
+///
+/// # Public Only for Internal Use:
+/// Record batch construction for tests in DataFusion crates. Not part of the supported
+/// public API.
+/// See the [API health policy] for details.
+///
+/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis
 #[deprecated(since = "55.0.0", note = "Use `arrow::array::record_batch` instead")]
 #[macro_export]
+#[doc(hidden)]
 macro_rules! record_batch {
     ($(($name: expr, $type: ident, $values: expr)),*) => {
         {

datafusion/core/Cargo.toml

@@ -39,6 +39,15 @@ all-features = true
 workspace = true
 
 [features]
+# Exposes internal test utilities for use by other DataFusion crates.
+test_utils = [
+    "datafusion-catalog/test_utils",
+    "datafusion-common/test_utils",
+    "datafusion-datasource/test_utils",
+    "datafusion-expr/test_utils",
+    "datafusion-physical-expr/test_utils",
+    "datafusion-physical-plan/test_utils",
+]
 # Enables the `print_config_docs`, `print_runtime_config_docs` and
 # `print_functions_docs` binaries, which regenerate the docs under
 # `docs/source/user-guide`. Off by default: they are only run by
@@ -171,6 +180,7 @@ async-trait = { workspace = true }
 criterion = { workspace = true, features = ["async_tokio", "async_futures"] }
 ctor = { workspace = true }
 dashmap = "6.2.1"
+datafusion = { workspace = true, features = ["test_utils"] }
 datafusion-doc = { workspace = true }
 datafusion-functions-window-common = { workspace = true }
 datafusion-macros = { workspace = true }

datafusion/core/src/lib.rs

@@ -854,8 +854,23 @@ pub mod physical_plan {
     pub use datafusion_physical_plan::*;
 }
 
-// Reexport testing macros for compatibility
+/// # Public Only for Internal Use:
+/// Test assertions re-exported for compatibility across DataFusion crates. Not part of
+/// the supported public API.
+/// See the [API health policy] for details.
+///
+/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis
+#[cfg(any(test, feature = "test_utils"))]
+#[doc(hidden)]
 pub use datafusion_common::assert_batches_eq;
+/// # Public Only for Internal Use:
+/// Test assertions re-exported for compatibility across DataFusion crates. Not part of
+/// the supported public API.
+/// See the [API health policy] for details.
+///
+/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis
+#[cfg(any(test, feature = "test_utils"))]
+#[doc(hidden)]
 pub use datafusion_common::assert_batches_sorted_eq;
 
 /// re-export of [`datafusion_sql`] crate
@@ -895,10 +910,24 @@ pub mod variable {
     pub use datafusion_expr::var_provider::{VarProvider, VarType};
 }
 
+/// # Public Only for Internal Use:
+/// Shared test helpers for DataFusion crates. Not part of the supported public API.
+/// See the [API health policy] for details.
+///
+/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis
 #[cfg(not(target_arch = "wasm32"))]
+#[cfg(any(test, feature = "test_utils"))]
+#[doc(hidden)]
 pub mod test;
 
 mod schema_equivalence;
+/// # Public Only for Internal Use:
+/// Shared test helpers for DataFusion crates. Not part of the supported public API.
+/// See the [API health policy] for details.
+///
+/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis
+#[cfg(any(test, feature = "test_utils"))]
+#[doc(hidden)]
 pub mod test_util;
 
 #[cfg(doctest)]

datafusion/datasource-avro/Cargo.toml

@@ -59,6 +59,7 @@ object_store = { workspace = true }
 tokio = { workspace = true }
 
 [dev-dependencies]
+datafusion-common = { workspace = true, features = ["test_utils"] }
 
 # Note: add additional linter rules in lib.rs.
 # Rust does not support workspace + new linter rules in subcrates yet

datafusion/datasource-parquet/Cargo.toml

@@ -60,6 +60,7 @@ tokio = { workspace = true }
 [dev-dependencies]
 chrono = { workspace = true }
 criterion = { workspace = true }
+datafusion-common = { workspace = true, features = ["test_utils"] }
 datafusion-functions = { workspace = true }
 datafusion-functions-nested = { workspace = true }
 tempfile = { workspace = true }

datafusion/datasource/Cargo.toml

@@ -31,6 +31,8 @@ version.workspace = true
 all-features = true
 
 [features]
+# Exposes internal test utilities for use by other DataFusion crates.
+test_utils = []
 backtrace = ["datafusion-common/backtrace"]
 compression = ["async-compression", "liblzma", "bzip2", "flate2", "zstd", "tokio-util"]
 default = ["compression"]
@@ -83,6 +85,8 @@ zstd = { workspace = true, optional = true }
 
 [dev-dependencies]
 criterion = { workspace = true }
+datafusion-common = { workspace = true, features = ["test_utils"] }
+datafusion-datasource = { workspace = true, features = ["test_utils"] }
 datafusion-functions = { workspace = true }
 insta = { workspace = true }
 tempfile = { workspace = true }

datafusion/datasource/src/mod.rs

@@ -58,10 +58,13 @@ pub mod url;
 pub mod write;
 pub use self::file::as_file_source;
 pub use self::url::ListingTableUrl;
+#[cfg(any(test, feature = "test_utils"))]
 use crate::file_groups::FileGroup;
 use arrow::datatypes::SchemaRef;
 use chrono::TimeZone;
-use datafusion_common::stats::{Precision, is_known_empty};
+use datafusion_common::stats::Precision;
+#[cfg(any(test, feature = "test_utils"))]
+use datafusion_common::stats::is_known_empty;
 use datafusion_common::{ColumnStatistics, Result, TableReference};
 use datafusion_common::{ScalarValue, Statistics};
 use datafusion_physical_expr::LexOrdering;
@@ -450,6 +453,14 @@ impl From<ObjectMeta> for PartitionedFile {
 /// File 2: [40, 140]
 /// File 3: [60, 160]
 /// File 4: [80, 180]
+///
+/// # Public Only for Internal Use:
+/// Used by file group tests and benchmarks. Not part of the supported public API.
+/// See the [API health policy] for details.
+///
+/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis
+#[cfg(any(test, feature = "test_utils"))]
+#[doc(hidden)]
 pub fn generate_test_files(num_files: usize, overlap_factor: f64) -> Vec<FileGroup> {
     let mut files = Vec::with_capacity(num_files);
     if num_files == 0 {
@@ -500,8 +511,15 @@ pub fn generate_test_files(num_files: usize, overlap_factor: f64) -> Vec<FileGro
     vec![FileGroup::new(files)]
 }
 
-// Helper function to verify that files within each group maintain sort order
-/// Used by tests and benchmarks
+/// Verifies that files within each group maintain sort order.
+///
+/// # Public Only for Internal Use:
+/// Used by file group tests and benchmarks. Not part of the supported public API.
+/// See the [API health policy] for details.
+///
+/// [API health policy]: https://datafusion.apache.org/contributor-guide/api-health.html#datafusion-internal-public-apis
+#[cfg(any(test, feature = "test_utils"))]
+#[doc(hidden)]
 pub fn verify_sort_integrity(file_groups: &[FileGroup]) -> bool {
     for group in file_groups {
         // Known-empty files contribute no rows and may not have min/max

datafusion/expr-common/Cargo.toml

@@ -47,5 +47,6 @@ indexmap = { workspace = true }
 itertools = { workspace = true }
 
 [dev-dependencies]
+datafusion-common = { workspace = true, features = ["test_utils"] }
 insta = { workspace = true }
 rstest = { workspace = true }

datafusion/expr/Cargo.toml

@@ -41,6 +41,8 @@ workspace = true
 name = "datafusion_expr"
 
 [features]
+# Exposes internal test utilities for use by other DataFusion crates.
+test_utils = []
 default = ["sql"]
 # Enables protobuf conversions for the expression types owned by this crate.
 # Off by default so consumers that never serialize plans pay nothing. Mirrors
@@ -71,6 +73,8 @@ sqlparser = { workspace = true, optional = true }
 [dev-dependencies]
 criterion = { workspace = true }
 ctor = { workspace = true }
+datafusion-common = { workspace = true, features = ["test_utils"] }
+datafusion-expr = { workspace = true, features = ["test_utils"] }
 env_logger = { workspace = true }
 insta = { workspace = true }
 # Makes sure `test_display_pg_json` behaves in a consistent way regardless of