From 2d47696a5f65e9fce10960d80b8bcba6cadbcb5c Mon Sep 17 00:00:00 2001 From: theirix Date: Sun, 19 Apr 2026 19:27:23 +0100 Subject: [PATCH 01/19] Use NativeType in get_example_types and information schema --- datafusion/catalog/src/information_schema.rs | 172 +++++++++++--- datafusion/expr-common/src/signature.rs | 230 +++++++------------ 2 files changed, 221 insertions(+), 181 deletions(-) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index 34c677c3dd43e..53b9609c42bad 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -22,16 +22,17 @@ use crate::streaming::StreamingTable; use crate::{CatalogProviderList, SchemaProvider, TableProvider}; use arrow::array::builder::{BooleanBuilder, UInt8Builder}; +use arrow::datatypes::{Fields, TimeUnit, UnionFields, UnionMode}; use arrow::{ array::{StringBuilder, UInt64Builder}, datatypes::{DataType, Field, FieldRef, Schema, SchemaRef}, record_batch::RecordBatch, }; use async_trait::async_trait; -use datafusion_common::DataFusionError; use datafusion_common::config::{ConfigEntry, ConfigOptions}; use datafusion_common::error::Result; use datafusion_common::types::NativeType; +use datafusion_common::{DataFusionError, not_impl_err}; use datafusion_execution::TaskContext; use datafusion_execution::runtime_env::RuntimeEnv; use datafusion_expr::function::WindowUDFFieldArgs; @@ -411,6 +412,127 @@ impl InformationSchemaConfig { } } +/// Resolve a native type `NativeType` to `DataType` for use in the information schema +/// Since it is one-to-many, use the most representative type on tie +fn get_data_type_for_schema(native_type: &NativeType) -> Option { + match native_type { + NativeType::Null => Some(DataType::Null), + NativeType::Boolean => Some(DataType::Boolean), + NativeType::Int8 => Some(DataType::Int8), + NativeType::Int16 => Some(DataType::Int16), + NativeType::Int32 => Some(DataType::Int32), + NativeType::Int64 => Some(DataType::Int64), + NativeType::UInt8 => Some(DataType::UInt8), + NativeType::UInt16 => Some(DataType::UInt16), + NativeType::UInt32 => Some(DataType::UInt32), + NativeType::UInt64 => Some(DataType::UInt64), + NativeType::Float16 => Some(DataType::Float16), + NativeType::Float32 => Some(DataType::Float32), + NativeType::Float64 => Some(DataType::Float64), + NativeType::Date => Some(DataType::Date32), // A tie + NativeType::Binary => Some(DataType::Binary), // A tie + NativeType::String => Some(DataType::Utf8), // A tie + NativeType::Decimal(precision, scale) => { + Some(DataType::Decimal256(*precision, *scale)) // A tie, use the widest type + } + NativeType::Timestamp(time_unit, timezone) => { + Some(DataType::Timestamp(*time_unit, timezone.to_owned())) + } + NativeType::Time(TimeUnit::Second) => Some(DataType::Time32(TimeUnit::Second)), + NativeType::Time(TimeUnit::Millisecond) => { + Some(DataType::Time32(TimeUnit::Millisecond)) + } + NativeType::Time(TimeUnit::Microsecond) => { + Some(DataType::Time64(TimeUnit::Microsecond)) + } + NativeType::Time(TimeUnit::Nanosecond) => { + Some(DataType::Time64(TimeUnit::Nanosecond)) + } + NativeType::Duration(time_unit) => Some(DataType::Duration(*time_unit)), + NativeType::Interval(interval_unit) => Some(DataType::Interval(*interval_unit)), + NativeType::FixedSizeBinary(size) => Some(DataType::FixedSizeBinary(*size)), + NativeType::FixedSizeList(logical_field, size) => get_data_type_for_schema( + logical_field.logical_type.native(), + ) + .map(|child_dt| { + DataType::FixedSizeList( + Arc::new(Field::new( + logical_field.name.clone(), + child_dt, + logical_field.nullable, + )), + *size, + ) + }), + NativeType::List(logical_field) => get_data_type_for_schema( + logical_field.logical_type.native(), + ) + .map(|child_dt| { + // A tie, use List + DataType::List(Arc::new(Field::new( + logical_field.name.clone(), + child_dt, + logical_field.nullable, + ))) + }), + NativeType::Struct(logical_fields) => { + let fields = logical_fields + .iter() + .map(|logical_field| { + let dt = + get_data_type_for_schema(logical_field.logical_type.native())?; + Some(Arc::new(Field::new( + logical_field.name.clone(), + dt, + logical_field.nullable, + ))) + }) + .collect::>()?; + Some(DataType::Struct(fields)) + } + NativeType::Union(logical_fields) => { + let ids = logical_fields.iter().map(|(i, _)| *i).collect::>(); + let fields: Vec = logical_fields + .iter() + .map(|(_, logical_field)| { + let dt = + get_data_type_for_schema(logical_field.logical_type.native())?; + Some(Arc::new(Field::new( + logical_field.name.clone(), + dt, + logical_field.nullable, + ))) + }) + .collect::>>()?; + Some(DataType::Union( + UnionFields::try_new(ids, fields).ok()?, + UnionMode::Dense, + )) + } + NativeType::Map(logical_field) => get_data_type_for_schema( + logical_field.logical_type.native(), + ) + .map(|child_dt| { + DataType::Map( + Arc::new(Field::new( + logical_field.name.clone(), + child_dt, + logical_field.nullable, + )), + true, + ) + }), + } +} + +pub fn resolve_informational_field(idx: usize, t: &NativeType) -> Result { + if let Some(data_type) = get_data_type_for_schema(t) { + Ok(Arc::new(Field::new(format!("arg_{idx}"), data_type, true))) + } else { + not_impl_err!("No support in information schema for type: {}", t) + } +} + /// get the arguments and return types of a UDF /// returns a tuple of (arg_types, return_type) fn get_udf_args_and_return_types( @@ -421,16 +543,14 @@ fn get_udf_args_and_return_types( if arg_types.is_empty() { Ok(vec![(vec![], None)].into_iter().collect::>()) } else { - Ok(arg_types + arg_types .into_iter() .map(|arg_types| { - let arg_fields: Vec = arg_types + let arg_fields = arg_types .iter() .enumerate() - .map(|(i, t)| { - Arc::new(Field::new(format!("arg_{i}"), t.clone(), true)) - }) - .collect(); + .map(|(i, t)| resolve_informational_field(i, t)) + .collect::>>()?; let scalar_arguments = vec![None; arg_fields.len()]; let return_type = udf .return_field_from_args(ReturnFieldArgs { @@ -445,11 +565,11 @@ fn get_udf_args_and_return_types( .ok(); let arg_types = arg_types .into_iter() - .map(|t| remove_native_type_prefix(&NativeType::from(t))) + .map(|t| remove_native_type_prefix(&t)) .collect::>(); - (arg_types, return_type) + Ok((arg_types, return_type)) }) - .collect::>()) + .collect::>>() } } @@ -461,16 +581,14 @@ fn get_udaf_args_and_return_types( if arg_types.is_empty() { Ok(vec![(vec![], None)].into_iter().collect::>()) } else { - Ok(arg_types + arg_types .into_iter() .map(|arg_types| { - let arg_fields: Vec = arg_types + let arg_fields = arg_types .iter() .enumerate() - .map(|(i, t)| { - Arc::new(Field::new(format!("arg_{i}"), t.clone(), true)) - }) - .collect(); + .map(|(i, t)| resolve_informational_field(i, t)) + .collect::>>()?; let return_type = udaf .return_field(&arg_fields) .map(|f| { @@ -481,11 +599,11 @@ fn get_udaf_args_and_return_types( .ok(); let arg_types = arg_types .into_iter() - .map(|t| remove_native_type_prefix(&NativeType::from(t))) + .map(|t| remove_native_type_prefix(&t)) .collect::>(); - (arg_types, return_type) + Ok((arg_types, return_type)) }) - .collect::>()) + .collect::>>() } } @@ -497,16 +615,14 @@ fn get_udwf_args_and_return_types( if arg_types.is_empty() { Ok(vec![(vec![], None)].into_iter().collect::>()) } else { - Ok(arg_types + arg_types .into_iter() .map(|arg_types| { - let arg_fields: Vec = arg_types + let arg_fields = arg_types .iter() .enumerate() - .map(|(i, t)| { - Arc::new(Field::new(format!("arg_{i}"), t.clone(), true)) - }) - .collect(); + .map(|(i, t)| resolve_informational_field(i, t)) + .collect::>>()?; let return_type = udwf .field(WindowUDFFieldArgs::new(&arg_fields, udwf.name())) .map(|f| { @@ -517,11 +633,11 @@ fn get_udwf_args_and_return_types( .ok(); let arg_types = arg_types .into_iter() - .map(|t| remove_native_type_prefix(&NativeType::from(t))) + .map(|t| remove_native_type_prefix(&t)) .collect::>(); - (arg_types, return_type) + Ok((arg_types, return_type)) }) - .collect::>()) + .collect::>>() } } diff --git a/datafusion/expr-common/src/signature.rs b/datafusion/expr-common/src/signature.rs index 3e941f00c2ee3..b89e37170ac64 100644 --- a/datafusion/expr-common/src/signature.rs +++ b/datafusion/expr-common/src/signature.rs @@ -19,11 +19,9 @@ use std::fmt::Display; use std::hash::Hash; -use std::sync::Arc; use arrow::datatypes::{ - DECIMAL32_MAX_PRECISION, DECIMAL64_MAX_PRECISION, DECIMAL128_MAX_PRECISION, DataType, - Decimal128Type, DecimalType, Field, IntervalUnit, TimeUnit, + DECIMAL256_MAX_PRECISION, DECIMAL256_MAX_SCALE, DataType, IntervalUnit, TimeUnit, }; use datafusion_common::types::{LogicalType, LogicalTypeRef, NativeType}; use datafusion_common::utils::ListCoercion; @@ -416,43 +414,39 @@ impl TypeSignatureClass { /// /// This is used for `information_schema` and can be used to generate /// documentation or error messages. - fn get_example_types(&self) -> Vec { + fn get_example_types(&self) -> Vec { match self { - // TODO: might be too much info to return every single type here - // maybe https://github.com/apache/datafusion/issues/14761 will help here? TypeSignatureClass::Any => vec![], - TypeSignatureClass::Native(l) => get_data_types(l.native()), + TypeSignatureClass::Native(l) => vec![l.native().clone()], TypeSignatureClass::Timestamp => { vec![ - DataType::Timestamp(TimeUnit::Nanosecond, None), - DataType::Timestamp( + NativeType::Timestamp(TimeUnit::Nanosecond, None), + NativeType::Timestamp( TimeUnit::Nanosecond, Some(TIMEZONE_WILDCARD.into()), ), ] } TypeSignatureClass::Time => { - vec![DataType::Time64(TimeUnit::Nanosecond)] + vec![NativeType::Time(TimeUnit::Nanosecond)] } TypeSignatureClass::Interval => { - vec![DataType::Interval(IntervalUnit::DayTime)] + vec![NativeType::Interval(IntervalUnit::DayTime)] } TypeSignatureClass::Duration => { - vec![DataType::Duration(TimeUnit::Nanosecond)] + vec![NativeType::Duration(TimeUnit::Nanosecond)] } TypeSignatureClass::Integer => { - vec![DataType::Int64] + vec![NativeType::Int64] } TypeSignatureClass::Binary => { - vec![DataType::Binary] + vec![NativeType::Binary] + } + TypeSignatureClass::Decimal => vec![NATIVE_TYPE_DECIMAL], + TypeSignatureClass::Float => vec![NativeType::Float64], + TypeSignatureClass::Numeric => { + vec![NativeType::Float64, NativeType::Int64, NATIVE_TYPE_DECIMAL] } - TypeSignatureClass::Decimal => vec![Decimal128Type::DEFAULT_TYPE], - TypeSignatureClass::Float => vec![DataType::Float64], - TypeSignatureClass::Numeric => vec![ - DataType::Float64, - DataType::Int64, - Decimal128Type::DEFAULT_TYPE, - ], } } @@ -595,18 +589,25 @@ impl Display for ArrayFunctionArgument { } } -static NUMERICS: &[DataType] = &[ - DataType::Int8, - DataType::Int16, - DataType::Int32, - DataType::Int64, - DataType::UInt8, - DataType::UInt16, - DataType::UInt32, - DataType::UInt64, - DataType::Float16, - DataType::Float32, - DataType::Float64, +/// Constant that is used as a Decimal type for `get_example_types` +/// Use Decimal256 precision as a reasonable default +const NATIVE_TYPE_DECIMAL: NativeType = + NativeType::Decimal(DECIMAL256_MAX_PRECISION, DECIMAL256_MAX_SCALE); + +/// Native types for `get_example_types` +static EXAMPLE_NUMERIC_TYPES: &[NativeType] = &[ + NativeType::Int8, + NativeType::Int16, + NativeType::Int32, + NativeType::Int64, + NativeType::UInt8, + NativeType::UInt16, + NativeType::UInt32, + NativeType::UInt64, + NativeType::Float16, + NativeType::Float32, + NativeType::Float64, + NATIVE_TYPE_DECIMAL, ]; impl TypeSignature { @@ -881,19 +882,24 @@ impl TypeSignature { } #[deprecated(since = "46.0.0", note = "See get_example_types instead")] - pub fn get_possible_types(&self) -> Vec> { + pub fn get_possible_types(&self) -> Vec> { self.get_example_types() } /// Return example acceptable types for this `TypeSignature`' /// - /// Returns a `Vec` for each argument to the function + /// Returns a `Vec` for each argument to the function /// /// This is used for `information_schema` and can be used to generate /// documentation or error messages. - pub fn get_example_types(&self) -> Vec> { + pub fn get_example_types(&self) -> Vec> { match self { - TypeSignature::Exact(types) => vec![types.clone()], + TypeSignature::Exact(types) => vec![ + types + .iter() + .map(|data_type| NativeType::from(data_type.clone())) + .collect(), + ], TypeSignature::OneOf(types) => types .iter() .flat_map(|type_sig| type_sig.get_example_types()) @@ -901,16 +907,16 @@ impl TypeSignature { TypeSignature::Uniform(arg_count, types) => types .iter() .cloned() - .map(|data_type| vec![data_type; *arg_count]) + .map(|data_type| vec![data_type.into(); *arg_count]) .collect(), TypeSignature::Coercible(coercions) => coercions .iter() .map(|c| { - let mut all_types: IndexSet = + let mut all_types: IndexSet = c.desired_type().get_example_types().into_iter().collect(); if let Some(implicit_coercion) = c.implicit_coercion() { - let allowed_casts: Vec = implicit_coercion + let allowed_casts: Vec = implicit_coercion .allowed_source_types .iter() .flat_map(|t| t.get_example_types()) @@ -925,17 +931,16 @@ impl TypeSignature { TypeSignature::Variadic(types) => types .iter() .cloned() - .map(|data_type| vec![data_type]) + .map(|data_type| vec![data_type.into()]) .collect(), - TypeSignature::Numeric(arg_count) => NUMERICS + TypeSignature::Numeric(arg_count) => EXAMPLE_NUMERIC_TYPES .iter() .cloned() .map(|numeric_type| vec![numeric_type; *arg_count]) .collect(), - TypeSignature::String(arg_count) => get_data_types(&NativeType::String) - .into_iter() - .map(|dt| vec![dt; *arg_count]) - .collect::>(), + TypeSignature::String(arg_count) => { + vec![vec![NativeType::String; *arg_count]] + } // TODO: Implement for other types TypeSignature::Any(_) | TypeSignature::Comparable(_) @@ -947,83 +952,6 @@ impl TypeSignature { } } -fn get_data_types(native_type: &NativeType) -> Vec { - match native_type { - NativeType::Null => vec![DataType::Null], - NativeType::Boolean => vec![DataType::Boolean], - NativeType::Int8 => vec![DataType::Int8], - NativeType::Int16 => vec![DataType::Int16], - NativeType::Int32 => vec![DataType::Int32], - NativeType::Int64 => vec![DataType::Int64], - NativeType::UInt8 => vec![DataType::UInt8], - NativeType::UInt16 => vec![DataType::UInt16], - NativeType::UInt32 => vec![DataType::UInt32], - NativeType::UInt64 => vec![DataType::UInt64], - NativeType::Float16 => vec![DataType::Float16], - NativeType::Float32 => vec![DataType::Float32], - NativeType::Float64 => vec![DataType::Float64], - NativeType::Date => vec![DataType::Date32, DataType::Date64], - NativeType::Binary => vec![ - DataType::Binary, - DataType::LargeBinary, - DataType::BinaryView, - ], - NativeType::String => { - vec![DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View] - } - NativeType::Decimal(precision, scale) => { - // We assume incoming NativeType is valid already, in terms of precision & scale - let mut types = vec![DataType::Decimal256(*precision, *scale)]; - if *precision <= DECIMAL32_MAX_PRECISION { - types.push(DataType::Decimal32(*precision, *scale)); - } - if *precision <= DECIMAL64_MAX_PRECISION { - types.push(DataType::Decimal64(*precision, *scale)); - } - if *precision <= DECIMAL128_MAX_PRECISION { - types.push(DataType::Decimal128(*precision, *scale)); - } - types - } - NativeType::Timestamp(time_unit, timezone) => { - vec![DataType::Timestamp(*time_unit, timezone.to_owned())] - } - NativeType::Time(TimeUnit::Second) => vec![DataType::Time32(TimeUnit::Second)], - NativeType::Time(TimeUnit::Millisecond) => { - vec![DataType::Time32(TimeUnit::Millisecond)] - } - NativeType::Time(TimeUnit::Microsecond) => { - vec![DataType::Time64(TimeUnit::Microsecond)] - } - NativeType::Time(TimeUnit::Nanosecond) => { - vec![DataType::Time64(TimeUnit::Nanosecond)] - } - NativeType::Duration(time_unit) => vec![DataType::Duration(*time_unit)], - NativeType::Interval(interval_unit) => vec![DataType::Interval(*interval_unit)], - NativeType::FixedSizeBinary(size) => vec![DataType::FixedSizeBinary(*size)], - NativeType::FixedSizeList(logical_field, size) => { - get_data_types(logical_field.logical_type.native()) - .iter() - .map(|child_dt| { - let field = Field::new( - logical_field.name.clone(), - child_dt.clone(), - logical_field.nullable, - ); - DataType::FixedSizeList(Arc::new(field), *size) - }) - .collect() - } - // TODO: implement for nested types - NativeType::List(_) - | NativeType::Struct(_) - | NativeType::Union(_) - | NativeType::Map(_) => { - vec![] - } - } -} - /// Represents type coercion rules for function arguments, specifying both the desired type /// and optional implicit coercion rules for source types. /// @@ -1572,7 +1500,10 @@ mod tests { fn test_get_possible_types() { let type_signature = TypeSignature::Exact(vec![DataType::Int32, DataType::Int64]); let possible_types = type_signature.get_example_types(); - assert_eq!(possible_types, vec![vec![DataType::Int32, DataType::Int64]]); + assert_eq!( + possible_types, + vec![vec![NativeType::Int32, NativeType::Int64]] + ); let type_signature = TypeSignature::OneOf(vec![ TypeSignature::Exact(vec![DataType::Int32, DataType::Int64]), @@ -1582,8 +1513,8 @@ mod tests { assert_eq!( possible_types, vec![ - vec![DataType::Int32, DataType::Int64], - vec![DataType::Float32, DataType::Float64] + vec![NativeType::Int32, NativeType::Int64], + vec![NativeType::Float32, NativeType::Float64] ] ); @@ -1596,9 +1527,9 @@ mod tests { assert_eq!( possible_types, vec![ - vec![DataType::Int32, DataType::Int64], - vec![DataType::Float32, DataType::Float64], - vec![DataType::Utf8] + vec![NativeType::Int32, NativeType::Int64], + vec![NativeType::Float32, NativeType::Float64], + vec![NativeType::String] ] ); @@ -1608,8 +1539,8 @@ mod tests { assert_eq!( possible_types, vec![ - vec![DataType::Float32, DataType::Float32], - vec![DataType::Int64, DataType::Int64] + vec![NativeType::Float32, NativeType::Float32], + vec![NativeType::Int64, NativeType::Int64] ] ); @@ -1620,11 +1551,7 @@ mod tests { let possible_types = type_signature.get_example_types(); assert_eq!( possible_types, - vec![ - vec![DataType::Utf8, DataType::Int64], - vec![DataType::LargeUtf8, DataType::Int64], - vec![DataType::Utf8View, DataType::Int64] - ] + vec![vec![NativeType::String, NativeType::Int64]] ); let type_signature = @@ -1632,7 +1559,7 @@ mod tests { let possible_types = type_signature.get_example_types(); assert_eq!( possible_types, - vec![vec![DataType::Int32], vec![DataType::Int64]] + vec![vec![NativeType::Int32], vec![NativeType::Int64]] ); let type_signature = TypeSignature::Numeric(2); @@ -1640,17 +1567,18 @@ mod tests { assert_eq!( possible_types, vec![ - vec![DataType::Int8, DataType::Int8], - vec![DataType::Int16, DataType::Int16], - vec![DataType::Int32, DataType::Int32], - vec![DataType::Int64, DataType::Int64], - vec![DataType::UInt8, DataType::UInt8], - vec![DataType::UInt16, DataType::UInt16], - vec![DataType::UInt32, DataType::UInt32], - vec![DataType::UInt64, DataType::UInt64], - vec![DataType::Float16, DataType::Float16], - vec![DataType::Float32, DataType::Float32], - vec![DataType::Float64, DataType::Float64] + vec![NativeType::Int8, NativeType::Int8], + vec![NativeType::Int16, NativeType::Int16], + vec![NativeType::Int32, NativeType::Int32], + vec![NativeType::Int64, NativeType::Int64], + vec![NativeType::UInt8, NativeType::UInt8], + vec![NativeType::UInt16, NativeType::UInt16], + vec![NativeType::UInt32, NativeType::UInt32], + vec![NativeType::UInt64, NativeType::UInt64], + vec![NativeType::Float16, NativeType::Float16], + vec![NativeType::Float32, NativeType::Float32], + vec![NativeType::Float64, NativeType::Float64], + vec![NATIVE_TYPE_DECIMAL, NATIVE_TYPE_DECIMAL], ] ); @@ -1658,11 +1586,7 @@ mod tests { let possible_types = type_signature.get_example_types(); assert_eq!( possible_types, - vec![ - vec![DataType::Utf8, DataType::Utf8], - vec![DataType::LargeUtf8, DataType::LargeUtf8], - vec![DataType::Utf8View, DataType::Utf8View] - ] + vec![vec![NativeType::String, NativeType::String],] ); } From 179f79417730a22698f56f04bb0d923bdb1a1049 Mon Sep 17 00:00:00 2001 From: theirix Date: Mon, 20 Apr 2026 20:08:40 +0300 Subject: [PATCH 02/19] Apply suggestions from code review Co-authored-by: Martin Grigorov --- datafusion/catalog/src/information_schema.rs | 2 +- datafusion/expr-common/src/signature.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index 53b9609c42bad..0e9da856a26db 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -525,7 +525,7 @@ fn get_data_type_for_schema(native_type: &NativeType) -> Option { } } -pub fn resolve_informational_field(idx: usize, t: &NativeType) -> Result { +fn resolve_informational_field(idx: usize, t: &NativeType) -> Result { if let Some(data_type) = get_data_type_for_schema(t) { Ok(Arc::new(Field::new(format!("arg_{idx}"), data_type, true))) } else { diff --git a/datafusion/expr-common/src/signature.rs b/datafusion/expr-common/src/signature.rs index b89e37170ac64..233c8272f12c2 100644 --- a/datafusion/expr-common/src/signature.rs +++ b/datafusion/expr-common/src/signature.rs @@ -592,7 +592,7 @@ impl Display for ArrayFunctionArgument { /// Constant that is used as a Decimal type for `get_example_types` /// Use Decimal256 precision as a reasonable default const NATIVE_TYPE_DECIMAL: NativeType = - NativeType::Decimal(DECIMAL256_MAX_PRECISION, DECIMAL256_MAX_SCALE); + NativeType::Decimal(DECIMAL256_MAX_PRECISION, DECIMAL_DEFAULT_SCALE); /// Native types for `get_example_types` static EXAMPLE_NUMERIC_TYPES: &[NativeType] = &[ From a22c2f6d9c030960f4b4e5b7a420d69dc50fea89 Mon Sep 17 00:00:00 2001 From: theirix Date: Mon, 20 Apr 2026 18:11:31 +0100 Subject: [PATCH 03/19] Use of Decimals --- datafusion/catalog/src/information_schema.rs | 2 +- datafusion/expr-common/src/signature.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index 0e9da856a26db..0679e330d43f4 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -433,7 +433,7 @@ fn get_data_type_for_schema(native_type: &NativeType) -> Option { NativeType::Binary => Some(DataType::Binary), // A tie NativeType::String => Some(DataType::Utf8), // A tie NativeType::Decimal(precision, scale) => { - Some(DataType::Decimal256(*precision, *scale)) // A tie, use the widest type + Some(DataType::Decimal128(*precision, *scale)) // A tie } NativeType::Timestamp(time_unit, timezone) => { Some(DataType::Timestamp(*time_unit, timezone.to_owned())) diff --git a/datafusion/expr-common/src/signature.rs b/datafusion/expr-common/src/signature.rs index 233c8272f12c2..f7c5de5490db0 100644 --- a/datafusion/expr-common/src/signature.rs +++ b/datafusion/expr-common/src/signature.rs @@ -21,7 +21,7 @@ use std::fmt::Display; use std::hash::Hash; use arrow::datatypes::{ - DECIMAL256_MAX_PRECISION, DECIMAL256_MAX_SCALE, DataType, IntervalUnit, TimeUnit, + DECIMAL_DEFAULT_SCALE, DECIMAL128_MAX_PRECISION, DataType, IntervalUnit, TimeUnit, }; use datafusion_common::types::{LogicalType, LogicalTypeRef, NativeType}; use datafusion_common::utils::ListCoercion; @@ -590,9 +590,9 @@ impl Display for ArrayFunctionArgument { } /// Constant that is used as a Decimal type for `get_example_types` -/// Use Decimal256 precision as a reasonable default +/// Use Decimal128 precision as a reasonable default const NATIVE_TYPE_DECIMAL: NativeType = - NativeType::Decimal(DECIMAL256_MAX_PRECISION, DECIMAL_DEFAULT_SCALE); + NativeType::Decimal(DECIMAL128_MAX_PRECISION, DECIMAL_DEFAULT_SCALE); /// Native types for `get_example_types` static EXAMPLE_NUMERIC_TYPES: &[NativeType] = &[ From c54da404918d680a09a241fe255cab95767de844 Mon Sep 17 00:00:00 2001 From: theirix Date: Mon, 20 Apr 2026 18:21:44 +0100 Subject: [PATCH 04/19] Refactor get_data_type_for_schema to return result --- datafusion/catalog/src/information_schema.rs | 113 +++++++++---------- 1 file changed, 52 insertions(+), 61 deletions(-) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index 0679e330d43f4..796e218ba7454 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -32,7 +32,7 @@ use async_trait::async_trait; use datafusion_common::config::{ConfigEntry, ConfigOptions}; use datafusion_common::error::Result; use datafusion_common::types::NativeType; -use datafusion_common::{DataFusionError, not_impl_err}; +use datafusion_common::{DataFusionError, internal_datafusion_err}; use datafusion_execution::TaskContext; use datafusion_execution::runtime_env::RuntimeEnv; use datafusion_expr::function::WindowUDFFieldArgs; @@ -414,81 +414,76 @@ impl InformationSchemaConfig { /// Resolve a native type `NativeType` to `DataType` for use in the information schema /// Since it is one-to-many, use the most representative type on tie -fn get_data_type_for_schema(native_type: &NativeType) -> Option { +fn get_data_type_for_schema(native_type: &NativeType) -> Result { match native_type { - NativeType::Null => Some(DataType::Null), - NativeType::Boolean => Some(DataType::Boolean), - NativeType::Int8 => Some(DataType::Int8), - NativeType::Int16 => Some(DataType::Int16), - NativeType::Int32 => Some(DataType::Int32), - NativeType::Int64 => Some(DataType::Int64), - NativeType::UInt8 => Some(DataType::UInt8), - NativeType::UInt16 => Some(DataType::UInt16), - NativeType::UInt32 => Some(DataType::UInt32), - NativeType::UInt64 => Some(DataType::UInt64), - NativeType::Float16 => Some(DataType::Float16), - NativeType::Float32 => Some(DataType::Float32), - NativeType::Float64 => Some(DataType::Float64), - NativeType::Date => Some(DataType::Date32), // A tie - NativeType::Binary => Some(DataType::Binary), // A tie - NativeType::String => Some(DataType::Utf8), // A tie + NativeType::Null => Ok(DataType::Null), + NativeType::Boolean => Ok(DataType::Boolean), + NativeType::Int8 => Ok(DataType::Int8), + NativeType::Int16 => Ok(DataType::Int16), + NativeType::Int32 => Ok(DataType::Int32), + NativeType::Int64 => Ok(DataType::Int64), + NativeType::UInt8 => Ok(DataType::UInt8), + NativeType::UInt16 => Ok(DataType::UInt16), + NativeType::UInt32 => Ok(DataType::UInt32), + NativeType::UInt64 => Ok(DataType::UInt64), + NativeType::Float16 => Ok(DataType::Float16), + NativeType::Float32 => Ok(DataType::Float32), + NativeType::Float64 => Ok(DataType::Float64), + NativeType::Date => Ok(DataType::Date32), // A tie + NativeType::Binary => Ok(DataType::Binary), // A tie + NativeType::String => Ok(DataType::Utf8), // A tie NativeType::Decimal(precision, scale) => { - Some(DataType::Decimal128(*precision, *scale)) // A tie + Ok(DataType::Decimal128(*precision, *scale)) // A tie } NativeType::Timestamp(time_unit, timezone) => { - Some(DataType::Timestamp(*time_unit, timezone.to_owned())) + Ok(DataType::Timestamp(*time_unit, timezone.to_owned())) } - NativeType::Time(TimeUnit::Second) => Some(DataType::Time32(TimeUnit::Second)), + NativeType::Time(TimeUnit::Second) => Ok(DataType::Time32(TimeUnit::Second)), NativeType::Time(TimeUnit::Millisecond) => { - Some(DataType::Time32(TimeUnit::Millisecond)) + Ok(DataType::Time32(TimeUnit::Millisecond)) } NativeType::Time(TimeUnit::Microsecond) => { - Some(DataType::Time64(TimeUnit::Microsecond)) + Ok(DataType::Time64(TimeUnit::Microsecond)) } NativeType::Time(TimeUnit::Nanosecond) => { - Some(DataType::Time64(TimeUnit::Nanosecond)) + Ok(DataType::Time64(TimeUnit::Nanosecond)) } - NativeType::Duration(time_unit) => Some(DataType::Duration(*time_unit)), - NativeType::Interval(interval_unit) => Some(DataType::Interval(*interval_unit)), - NativeType::FixedSizeBinary(size) => Some(DataType::FixedSizeBinary(*size)), - NativeType::FixedSizeList(logical_field, size) => get_data_type_for_schema( - logical_field.logical_type.native(), - ) - .map(|child_dt| { - DataType::FixedSizeList( + NativeType::Duration(time_unit) => Ok(DataType::Duration(*time_unit)), + NativeType::Interval(interval_unit) => Ok(DataType::Interval(*interval_unit)), + NativeType::FixedSizeBinary(size) => Ok(DataType::FixedSizeBinary(*size)), + NativeType::FixedSizeList(logical_field, size) => { + let child_dt = get_data_type_for_schema(logical_field.logical_type.native())?; + Ok(DataType::FixedSizeList( Arc::new(Field::new( logical_field.name.clone(), child_dt, logical_field.nullable, )), *size, - ) - }), - NativeType::List(logical_field) => get_data_type_for_schema( - logical_field.logical_type.native(), - ) - .map(|child_dt| { - // A tie, use List - DataType::List(Arc::new(Field::new( + )) + } + NativeType::List(logical_field) => { + let child_dt = get_data_type_for_schema(logical_field.logical_type.native())?; + Ok(DataType::List(Arc::new(Field::new( logical_field.name.clone(), child_dt, logical_field.nullable, - ))) - }), + )))) + } NativeType::Struct(logical_fields) => { let fields = logical_fields .iter() .map(|logical_field| { let dt = get_data_type_for_schema(logical_field.logical_type.native())?; - Some(Arc::new(Field::new( + Ok(Arc::new(Field::new( logical_field.name.clone(), dt, logical_field.nullable, ))) }) - .collect::>()?; - Some(DataType::Struct(fields)) + .collect::>()?; + Ok(DataType::Struct(fields)) } NativeType::Union(logical_fields) => { let ids = logical_fields.iter().map(|(i, _)| *i).collect::>(); @@ -497,40 +492,36 @@ fn get_data_type_for_schema(native_type: &NativeType) -> Option { .map(|(_, logical_field)| { let dt = get_data_type_for_schema(logical_field.logical_type.native())?; - Some(Arc::new(Field::new( + Ok(Arc::new(Field::new( logical_field.name.clone(), dt, logical_field.nullable, ))) }) - .collect::>>()?; - Some(DataType::Union( - UnionFields::try_new(ids, fields).ok()?, + .collect::>>()?; + Ok(DataType::Union( + UnionFields::try_new(ids, fields) + .map_err(|e| internal_datafusion_err!("UnionFields error: {e}"))?, UnionMode::Dense, )) } - NativeType::Map(logical_field) => get_data_type_for_schema( - logical_field.logical_type.native(), - ) - .map(|child_dt| { - DataType::Map( + NativeType::Map(logical_field) => { + let child_dt = get_data_type_for_schema(logical_field.logical_type.native())?; + Ok(DataType::Map( Arc::new(Field::new( logical_field.name.clone(), child_dt, logical_field.nullable, )), true, - ) - }), + )) + } } } fn resolve_informational_field(idx: usize, t: &NativeType) -> Result { - if let Some(data_type) = get_data_type_for_schema(t) { - Ok(Arc::new(Field::new(format!("arg_{idx}"), data_type, true))) - } else { - not_impl_err!("No support in information schema for type: {}", t) - } + let data_type = get_data_type_for_schema(t)?; + Ok(Arc::new(Field::new(format!("arg_{idx}"), data_type, true))) } /// get the arguments and return types of a UDF From 50b6858f655a245e60a077a4f1b8852b3e1e5ca7 Mon Sep 17 00:00:00 2001 From: theirix Date: Tue, 21 Apr 2026 21:02:09 +0100 Subject: [PATCH 05/19] Remove old deprecated get_possible_types --- datafusion/expr-common/src/signature.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/datafusion/expr-common/src/signature.rs b/datafusion/expr-common/src/signature.rs index f7c5de5490db0..ac5462e8809a2 100644 --- a/datafusion/expr-common/src/signature.rs +++ b/datafusion/expr-common/src/signature.rs @@ -881,10 +881,6 @@ impl TypeSignature { } } - #[deprecated(since = "46.0.0", note = "See get_example_types instead")] - pub fn get_possible_types(&self) -> Vec> { - self.get_example_types() - } /// Return example acceptable types for this `TypeSignature`' /// From 52769334a2b9c5dd48bafd1949279ba695ed2951 Mon Sep 17 00:00:00 2001 From: theirix Date: Tue, 21 Apr 2026 21:02:49 +0100 Subject: [PATCH 06/19] Avoid breaking get_example_types API Instead, add the new `get_representative_types` API with NativeType. Deprecated old `get_example_types` and related helpers, left as-is to avoid breaking API change. --- datafusion/catalog/src/information_schema.rs | 6 +- datafusion/expr-common/src/signature.rs | 344 +++++++++++++++++-- 2 files changed, 325 insertions(+), 25 deletions(-) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index 796e218ba7454..51d1e73857ee3 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -530,7 +530,7 @@ fn get_udf_args_and_return_types( udf: &Arc, ) -> Result, Option)>> { let signature = udf.signature(); - let arg_types = signature.type_signature.get_example_types(); + let arg_types = signature.type_signature.get_representative_types(); if arg_types.is_empty() { Ok(vec![(vec![], None)].into_iter().collect::>()) } else { @@ -568,7 +568,7 @@ fn get_udaf_args_and_return_types( udaf: &Arc, ) -> Result, Option)>> { let signature = udaf.signature(); - let arg_types = signature.type_signature.get_example_types(); + let arg_types = signature.type_signature.get_representative_types(); if arg_types.is_empty() { Ok(vec![(vec![], None)].into_iter().collect::>()) } else { @@ -602,7 +602,7 @@ fn get_udwf_args_and_return_types( udwf: &Arc, ) -> Result, Option)>> { let signature = udwf.signature(); - let arg_types = signature.type_signature.get_example_types(); + let arg_types = signature.type_signature.get_representative_types(); if arg_types.is_empty() { Ok(vec![(vec![], None)].into_iter().collect::>()) } else { diff --git a/datafusion/expr-common/src/signature.rs b/datafusion/expr-common/src/signature.rs index ac5462e8809a2..ffac009aa4952 100644 --- a/datafusion/expr-common/src/signature.rs +++ b/datafusion/expr-common/src/signature.rs @@ -19,9 +19,12 @@ use std::fmt::Display; use std::hash::Hash; +use std::sync::Arc; use arrow::datatypes::{ - DECIMAL_DEFAULT_SCALE, DECIMAL128_MAX_PRECISION, DataType, IntervalUnit, TimeUnit, + DECIMAL_DEFAULT_SCALE, DECIMAL32_MAX_PRECISION, DECIMAL64_MAX_PRECISION, + DECIMAL128_MAX_PRECISION, DataType, Decimal128Type, DecimalType, Field, IntervalUnit, + TimeUnit, }; use datafusion_common::types::{LogicalType, LogicalTypeRef, NativeType}; use datafusion_common::utils::ListCoercion; @@ -414,7 +417,52 @@ impl TypeSignatureClass { /// /// This is used for `information_schema` and can be used to generate /// documentation or error messages. - fn get_example_types(&self) -> Vec { + /// Remove with `get_example_types` + #[deprecated(since = "53.0.0", note = "See get_representative_types instead")] + fn get_example_types(&self) -> Vec { + match self { + TypeSignatureClass::Any => vec![], + #[expect(deprecated)] + TypeSignatureClass::Native(l) => get_data_types(l.native()), + TypeSignatureClass::Timestamp => { + vec![ + DataType::Timestamp(TimeUnit::Nanosecond, None), + DataType::Timestamp( + TimeUnit::Nanosecond, + Some(TIMEZONE_WILDCARD.into()), + ), + ] + } + TypeSignatureClass::Time => { + vec![DataType::Time64(TimeUnit::Nanosecond)] + } + TypeSignatureClass::Interval => { + vec![DataType::Interval(IntervalUnit::DayTime)] + } + TypeSignatureClass::Duration => { + vec![DataType::Duration(TimeUnit::Nanosecond)] + } + TypeSignatureClass::Integer => { + vec![DataType::Int64] + } + TypeSignatureClass::Binary => { + vec![DataType::Binary] + } + TypeSignatureClass::Decimal => vec![Decimal128Type::DEFAULT_TYPE], + TypeSignatureClass::Float => vec![DataType::Float64], + TypeSignatureClass::Numeric => vec![ + DataType::Float64, + DataType::Int64, + Decimal128Type::DEFAULT_TYPE, + ], + } + } + + /// Get example acceptable types for this `TypeSignatureClass` + /// + /// This is used for `information_schema` and can be used to generate + /// documentation or error messages. + fn get_representative_types(&self) -> Vec { match self { TypeSignatureClass::Any => vec![], TypeSignatureClass::Native(l) => vec![l.native().clone()], @@ -610,6 +658,21 @@ static EXAMPLE_NUMERIC_TYPES: &[NativeType] = &[ NATIVE_TYPE_DECIMAL, ]; +#[deprecated(since = "53.0.0", note = "See get_representative_types instead")] +static NUMERICS: &[DataType] = &[ + DataType::Int8, + DataType::Int16, + DataType::Int32, + DataType::Int64, + DataType::UInt8, + DataType::UInt16, + DataType::UInt32, + DataType::UInt64, + DataType::Float16, + DataType::Float32, + DataType::Float64, +]; + impl TypeSignature { pub fn to_string_repr(&self) -> Vec { match self { @@ -881,6 +944,68 @@ impl TypeSignature { } } + /// Return example acceptable types for this `TypeSignature`' + /// + /// Returns a `Vec` for each argument to the function + /// + /// This is used for `information_schema` and can be used to generate + /// documentation or error messages. + #[deprecated(since = "53.0.0", note = "See get_representative_types instead")] + pub fn get_example_types(&self) -> Vec> { + #[expect(deprecated)] + match self { + TypeSignature::Exact(types) => vec![types.clone()], + TypeSignature::OneOf(types) => types + .iter() + .flat_map(|type_sig| type_sig.get_example_types()) + .collect(), + TypeSignature::Uniform(arg_count, types) => types + .iter() + .cloned() + .map(|data_type| vec![data_type; *arg_count]) + .collect(), + TypeSignature::Coercible(coercions) => coercions + .iter() + .map(|c| { + let mut all_types: IndexSet = + c.desired_type().get_example_types().into_iter().collect(); + + if let Some(implicit_coercion) = c.implicit_coercion() { + let allowed_casts: Vec = implicit_coercion + .allowed_source_types + .iter() + .flat_map(|t| t.get_example_types()) + .collect(); + all_types.extend(allowed_casts); + } + + all_types.into_iter().collect::>() + }) + .multi_cartesian_product() + .collect(), + TypeSignature::Variadic(types) => types + .iter() + .cloned() + .map(|data_type| vec![data_type]) + .collect(), + TypeSignature::Numeric(arg_count) => NUMERICS + .iter() + .cloned() + .map(|numeric_type| vec![numeric_type; *arg_count]) + .collect(), + TypeSignature::String(arg_count) => get_data_types(&NativeType::String) + .into_iter() + .map(|dt| vec![dt; *arg_count]) + .collect::>(), + // TODO: Implement for other types + TypeSignature::Any(_) + | TypeSignature::Comparable(_) + | TypeSignature::Nullary + | TypeSignature::VariadicAny + | TypeSignature::ArraySignature(_) + | TypeSignature::UserDefined => vec![], + } + } /// Return example acceptable types for this `TypeSignature`' /// @@ -888,7 +1013,7 @@ impl TypeSignature { /// /// This is used for `information_schema` and can be used to generate /// documentation or error messages. - pub fn get_example_types(&self) -> Vec> { + pub fn get_representative_types(&self) -> Vec> { match self { TypeSignature::Exact(types) => vec![ types @@ -898,7 +1023,7 @@ impl TypeSignature { ], TypeSignature::OneOf(types) => types .iter() - .flat_map(|type_sig| type_sig.get_example_types()) + .flat_map(|type_sig| type_sig.get_representative_types()) .collect(), TypeSignature::Uniform(arg_count, types) => types .iter() @@ -908,14 +1033,17 @@ impl TypeSignature { TypeSignature::Coercible(coercions) => coercions .iter() .map(|c| { - let mut all_types: IndexSet = - c.desired_type().get_example_types().into_iter().collect(); + let mut all_types: IndexSet = c + .desired_type() + .get_representative_types() + .into_iter() + .collect(); if let Some(implicit_coercion) = c.implicit_coercion() { let allowed_casts: Vec = implicit_coercion .allowed_source_types .iter() - .flat_map(|t| t.get_example_types()) + .flat_map(|t| t.get_representative_types()) .collect(); all_types.extend(allowed_casts); } @@ -948,6 +1076,84 @@ impl TypeSignature { } } +#[deprecated(since = "53.0.0", note = "See get_representative_types instead")] +fn get_data_types(native_type: &NativeType) -> Vec { + match native_type { + NativeType::Null => vec![DataType::Null], + NativeType::Boolean => vec![DataType::Boolean], + NativeType::Int8 => vec![DataType::Int8], + NativeType::Int16 => vec![DataType::Int16], + NativeType::Int32 => vec![DataType::Int32], + NativeType::Int64 => vec![DataType::Int64], + NativeType::UInt8 => vec![DataType::UInt8], + NativeType::UInt16 => vec![DataType::UInt16], + NativeType::UInt32 => vec![DataType::UInt32], + NativeType::UInt64 => vec![DataType::UInt64], + NativeType::Float16 => vec![DataType::Float16], + NativeType::Float32 => vec![DataType::Float32], + NativeType::Float64 => vec![DataType::Float64], + NativeType::Date => vec![DataType::Date32, DataType::Date64], + NativeType::Binary => vec![ + DataType::Binary, + DataType::LargeBinary, + DataType::BinaryView, + ], + NativeType::String => { + vec![DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View] + } + NativeType::Decimal(precision, scale) => { + // We assume incoming NativeType is valid already, in terms of precision & scale + let mut types = vec![DataType::Decimal256(*precision, *scale)]; + if *precision <= DECIMAL32_MAX_PRECISION { + types.push(DataType::Decimal32(*precision, *scale)); + } + if *precision <= DECIMAL64_MAX_PRECISION { + types.push(DataType::Decimal64(*precision, *scale)); + } + if *precision <= DECIMAL128_MAX_PRECISION { + types.push(DataType::Decimal128(*precision, *scale)); + } + types + } + NativeType::Timestamp(time_unit, timezone) => { + vec![DataType::Timestamp(*time_unit, timezone.to_owned())] + } + NativeType::Time(TimeUnit::Second) => vec![DataType::Time32(TimeUnit::Second)], + NativeType::Time(TimeUnit::Millisecond) => { + vec![DataType::Time32(TimeUnit::Millisecond)] + } + NativeType::Time(TimeUnit::Microsecond) => { + vec![DataType::Time64(TimeUnit::Microsecond)] + } + NativeType::Time(TimeUnit::Nanosecond) => { + vec![DataType::Time64(TimeUnit::Nanosecond)] + } + NativeType::Duration(time_unit) => vec![DataType::Duration(*time_unit)], + NativeType::Interval(interval_unit) => vec![DataType::Interval(*interval_unit)], + NativeType::FixedSizeBinary(size) => vec![DataType::FixedSizeBinary(*size)], + NativeType::FixedSizeList(logical_field, size) => { + get_data_types(logical_field.logical_type.native()) + .iter() + .map(|child_dt| { + let field = Field::new( + logical_field.name.clone(), + child_dt.clone(), + logical_field.nullable, + ); + DataType::FixedSizeList(Arc::new(field), *size) + }) + .collect() + } + // TODO: implement for nested types + NativeType::List(_) + | NativeType::Struct(_) + | NativeType::Union(_) + | NativeType::Map(_) => { + vec![] + } + } +} + /// Represents type coercion rules for function arguments, specifying both the desired type /// and optional implicit coercion rules for source types. /// @@ -1492,14 +1698,13 @@ mod tests { ); } + // Remove with get_example_types #[test] - fn test_get_possible_types() { + #[expect(deprecated)] + fn test_get_example_types() { let type_signature = TypeSignature::Exact(vec![DataType::Int32, DataType::Int64]); let possible_types = type_signature.get_example_types(); - assert_eq!( - possible_types, - vec![vec![NativeType::Int32, NativeType::Int64]] - ); + assert_eq!(possible_types, vec![vec![DataType::Int32, DataType::Int64]]); let type_signature = TypeSignature::OneOf(vec![ TypeSignature::Exact(vec![DataType::Int32, DataType::Int64]), @@ -1509,8 +1714,8 @@ mod tests { assert_eq!( possible_types, vec![ - vec![NativeType::Int32, NativeType::Int64], - vec![NativeType::Float32, NativeType::Float64] + vec![DataType::Int32, DataType::Int64], + vec![DataType::Float32, DataType::Float64] ] ); @@ -1523,9 +1728,9 @@ mod tests { assert_eq!( possible_types, vec![ - vec![NativeType::Int32, NativeType::Int64], - vec![NativeType::Float32, NativeType::Float64], - vec![NativeType::String] + vec![DataType::Int32, DataType::Int64], + vec![DataType::Float32, DataType::Float64], + vec![DataType::Utf8] ] ); @@ -1535,8 +1740,8 @@ mod tests { assert_eq!( possible_types, vec![ - vec![NativeType::Float32, NativeType::Float32], - vec![NativeType::Int64, NativeType::Int64] + vec![DataType::Float32, DataType::Float32], + vec![DataType::Int64, DataType::Int64] ] ); @@ -1547,7 +1752,11 @@ mod tests { let possible_types = type_signature.get_example_types(); assert_eq!( possible_types, - vec![vec![NativeType::String, NativeType::Int64]] + vec![ + vec![DataType::Utf8, DataType::Int64], + vec![DataType::LargeUtf8, DataType::Int64], + vec![DataType::Utf8View, DataType::Int64] + ] ); let type_signature = @@ -1555,13 +1764,104 @@ mod tests { let possible_types = type_signature.get_example_types(); assert_eq!( possible_types, - vec![vec![NativeType::Int32], vec![NativeType::Int64]] + vec![vec![DataType::Int32], vec![DataType::Int64]] ); let type_signature = TypeSignature::Numeric(2); let possible_types = type_signature.get_example_types(); assert_eq!( possible_types, + vec![ + vec![DataType::Int8, DataType::Int8], + vec![DataType::Int16, DataType::Int16], + vec![DataType::Int32, DataType::Int32], + vec![DataType::Int64, DataType::Int64], + vec![DataType::UInt8, DataType::UInt8], + vec![DataType::UInt16, DataType::UInt16], + vec![DataType::UInt32, DataType::UInt32], + vec![DataType::UInt64, DataType::UInt64], + vec![DataType::Float16, DataType::Float16], + vec![DataType::Float32, DataType::Float32], + vec![DataType::Float64, DataType::Float64] + ] + ); + + let type_signature = TypeSignature::String(2); + let possible_types = type_signature.get_example_types(); + assert_eq!( + possible_types, + vec![ + vec![DataType::Utf8, DataType::Utf8], + vec![DataType::LargeUtf8, DataType::LargeUtf8], + vec![DataType::Utf8View, DataType::Utf8View] + ] + ); + } + + #[test] + fn test_get_representative_types() { + let type_signature = TypeSignature::Exact(vec![DataType::Int32, DataType::Int64]); + let types = type_signature.get_representative_types(); + assert_eq!(types, vec![vec![NativeType::Int32, NativeType::Int64]]); + + let type_signature = TypeSignature::OneOf(vec![ + TypeSignature::Exact(vec![DataType::Int32, DataType::Int64]), + TypeSignature::Exact(vec![DataType::Float32, DataType::Float64]), + ]); + let types = type_signature.get_representative_types(); + assert_eq!( + types, + vec![ + vec![NativeType::Int32, NativeType::Int64], + vec![NativeType::Float32, NativeType::Float64] + ] + ); + + let type_signature = TypeSignature::OneOf(vec![ + TypeSignature::Exact(vec![DataType::Int32, DataType::Int64]), + TypeSignature::Exact(vec![DataType::Float32, DataType::Float64]), + TypeSignature::Exact(vec![DataType::Utf8]), + ]); + let types = type_signature.get_representative_types(); + assert_eq!( + types, + vec![ + vec![NativeType::Int32, NativeType::Int64], + vec![NativeType::Float32, NativeType::Float64], + vec![NativeType::String] + ] + ); + + let type_signature = + TypeSignature::Uniform(2, vec![DataType::Float32, DataType::Int64]); + let types = type_signature.get_representative_types(); + assert_eq!( + types, + vec![ + vec![NativeType::Float32, NativeType::Float32], + vec![NativeType::Int64, NativeType::Int64] + ] + ); + + let type_signature = TypeSignature::Coercible(vec![ + Coercion::new_exact(TypeSignatureClass::Native(logical_string())), + Coercion::new_exact(TypeSignatureClass::Native(logical_int64())), + ]); + let types = type_signature.get_representative_types(); + assert_eq!(types, vec![vec![NativeType::String, NativeType::Int64]]); + + let type_signature = + TypeSignature::Variadic(vec![DataType::Int32, DataType::Int64]); + let types = type_signature.get_representative_types(); + assert_eq!( + types, + vec![vec![NativeType::Int32], vec![NativeType::Int64]] + ); + + let type_signature = TypeSignature::Numeric(2); + let types = type_signature.get_representative_types(); + assert_eq!( + types, vec![ vec![NativeType::Int8, NativeType::Int8], vec![NativeType::Int16, NativeType::Int16], @@ -1579,7 +1879,7 @@ mod tests { ); let type_signature = TypeSignature::String(2); - let possible_types = type_signature.get_example_types(); + let possible_types = type_signature.get_representative_types(); assert_eq!( possible_types, vec![vec![NativeType::String, NativeType::String],] From 840fbb99d7b149bb7774618eaf0ed15ea4fb5ae2 Mon Sep 17 00:00:00 2001 From: theirix Date: Wed, 22 Apr 2026 18:01:20 +0100 Subject: [PATCH 07/19] Fix docs --- datafusion/expr-common/src/signature.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/expr-common/src/signature.rs b/datafusion/expr-common/src/signature.rs index ffac009aa4952..97010ca286ed8 100644 --- a/datafusion/expr-common/src/signature.rs +++ b/datafusion/expr-common/src/signature.rs @@ -417,7 +417,7 @@ impl TypeSignatureClass { /// /// This is used for `information_schema` and can be used to generate /// documentation or error messages. - /// Remove with `get_example_types` + // Remove with `get_example_types` #[deprecated(since = "53.0.0", note = "See get_representative_types instead")] fn get_example_types(&self) -> Vec { match self { From 687fa6a4c4df22c1795c2f4d37b9403e5c1483fd Mon Sep 17 00:00:00 2001 From: theirix Date: Tue, 18 Aug 2026 09:29:26 +0100 Subject: [PATCH 08/19] Reuse LogicalType type mapping --- datafusion/catalog/src/information_schema.rs | 132 +++--------------- datafusion/common/src/types/native.rs | 124 +++++++++++++++- datafusion/expr-common/src/signature.rs | 74 ++++------ .../test_files/information_schema.slt | 51 +++++++ 4 files changed, 216 insertions(+), 165 deletions(-) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index 4b76cc56a8a39..d0a29d513f7d6 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -23,17 +23,16 @@ use crate::streaming::StreamingTable; use crate::table::TableFunction; use crate::{CatalogProviderList, SchemaProvider, TableProvider}; use arrow::array::builder::{BooleanBuilder, UInt8Builder}; -use arrow::datatypes::{Fields, TimeUnit, UnionFields, UnionMode}; use arrow::{ array::{StringBuilder, UInt64Builder}, datatypes::{DataType, Field, FieldRef, Schema, SchemaRef}, record_batch::RecordBatch, }; use async_trait::async_trait; +use datafusion_common::DataFusionError; use datafusion_common::config::{ConfigEntry, ConfigOptions}; use datafusion_common::error::Result; -use datafusion_common::types::NativeType; -use datafusion_common::{DataFusionError, internal_datafusion_err}; +use datafusion_common::types::{LogicalType, NativeType}; use datafusion_execution::TaskContext; use datafusion_execution::runtime_env::RuntimeEnv; use datafusion_expr::function::WindowUDFFieldArgs; @@ -455,115 +454,11 @@ impl InformationSchemaConfig { } } -/// Resolve a native type `NativeType` to `DataType` for use in the information schema -/// Since it is one-to-many, use the most representative type on tie -fn get_data_type_for_schema(native_type: &NativeType) -> Result { - match native_type { - NativeType::Null => Ok(DataType::Null), - NativeType::Boolean => Ok(DataType::Boolean), - NativeType::Int8 => Ok(DataType::Int8), - NativeType::Int16 => Ok(DataType::Int16), - NativeType::Int32 => Ok(DataType::Int32), - NativeType::Int64 => Ok(DataType::Int64), - NativeType::UInt8 => Ok(DataType::UInt8), - NativeType::UInt16 => Ok(DataType::UInt16), - NativeType::UInt32 => Ok(DataType::UInt32), - NativeType::UInt64 => Ok(DataType::UInt64), - NativeType::Float16 => Ok(DataType::Float16), - NativeType::Float32 => Ok(DataType::Float32), - NativeType::Float64 => Ok(DataType::Float64), - NativeType::Date => Ok(DataType::Date32), // A tie - NativeType::Binary => Ok(DataType::Binary), // A tie - NativeType::String => Ok(DataType::Utf8), // A tie - NativeType::Decimal(precision, scale) => { - Ok(DataType::Decimal128(*precision, *scale)) // A tie - } - NativeType::Timestamp(time_unit, timezone) => { - Ok(DataType::Timestamp(*time_unit, timezone.to_owned())) - } - NativeType::Time(TimeUnit::Second) => Ok(DataType::Time32(TimeUnit::Second)), - NativeType::Time(TimeUnit::Millisecond) => { - Ok(DataType::Time32(TimeUnit::Millisecond)) - } - NativeType::Time(TimeUnit::Microsecond) => { - Ok(DataType::Time64(TimeUnit::Microsecond)) - } - NativeType::Time(TimeUnit::Nanosecond) => { - Ok(DataType::Time64(TimeUnit::Nanosecond)) - } - NativeType::Duration(time_unit) => Ok(DataType::Duration(*time_unit)), - NativeType::Interval(interval_unit) => Ok(DataType::Interval(*interval_unit)), - NativeType::FixedSizeBinary(size) => Ok(DataType::FixedSizeBinary(*size)), - NativeType::FixedSizeList(logical_field, size) => { - let child_dt = get_data_type_for_schema(logical_field.logical_type.native())?; - Ok(DataType::FixedSizeList( - Arc::new(Field::new( - logical_field.name.clone(), - child_dt, - logical_field.nullable, - )), - *size, - )) - } - NativeType::List(logical_field) => { - let child_dt = get_data_type_for_schema(logical_field.logical_type.native())?; - Ok(DataType::List(Arc::new(Field::new( - logical_field.name.clone(), - child_dt, - logical_field.nullable, - )))) - } - NativeType::Struct(logical_fields) => { - let fields = logical_fields - .iter() - .map(|logical_field| { - let dt = - get_data_type_for_schema(logical_field.logical_type.native())?; - Ok(Arc::new(Field::new( - logical_field.name.clone(), - dt, - logical_field.nullable, - ))) - }) - .collect::>()?; - Ok(DataType::Struct(fields)) - } - NativeType::Union(logical_fields) => { - let ids = logical_fields.iter().map(|(i, _)| *i).collect::>(); - let fields: Vec = logical_fields - .iter() - .map(|(_, logical_field)| { - let dt = - get_data_type_for_schema(logical_field.logical_type.native())?; - Ok(Arc::new(Field::new( - logical_field.name.clone(), - dt, - logical_field.nullable, - ))) - }) - .collect::>>()?; - Ok(DataType::Union( - UnionFields::try_new(ids, fields) - .map_err(|e| internal_datafusion_err!("UnionFields error: {e}"))?, - UnionMode::Dense, - )) - } - NativeType::Map(logical_field) => { - let child_dt = get_data_type_for_schema(logical_field.logical_type.native())?; - Ok(DataType::Map( - Arc::new(Field::new( - logical_field.name.clone(), - child_dt, - logical_field.nullable, - )), - true, - )) - } - } -} - +/// Build the argument field for `information_schema` to provide a return type fn resolve_informational_field(idx: usize, t: &NativeType) -> Result { - let data_type = get_data_type_for_schema(t)?; + // Since a native type maps to several physical types, resolve it against `Null` data type + // to get the canonical `DataType` for the native type + let data_type = t.default_cast_for(&DataType::Null)?; Ok(Arc::new(Field::new(format!("arg_{idx}"), data_type, true))) } @@ -580,11 +475,14 @@ fn get_udf_args_and_return_types( arg_types .into_iter() .map(|arg_types| { - let arg_fields = arg_types + let mut arg_fields = arg_types .iter() .enumerate() .map(|(i, t)| resolve_informational_field(i, t)) .collect::>>()?; + // Even with collecting results into a set, drop duplicates early + arg_fields.sort(); + arg_fields.dedup(); let scalar_arguments = vec![None; arg_fields.len()]; let return_type = udf .return_field_from_args(ReturnFieldArgs { @@ -618,11 +516,14 @@ fn get_udaf_args_and_return_types( arg_types .into_iter() .map(|arg_types| { - let arg_fields = arg_types + let mut arg_fields = arg_types .iter() .enumerate() .map(|(i, t)| resolve_informational_field(i, t)) .collect::>>()?; + // Even with collecting results into a set, drop duplicates early + arg_fields.sort(); + arg_fields.dedup(); let return_type = udaf .return_field(&arg_fields) .map(|f| { @@ -652,11 +553,14 @@ fn get_udwf_args_and_return_types( arg_types .into_iter() .map(|arg_types| { - let arg_fields = arg_types + let mut arg_fields = arg_types .iter() .enumerate() .map(|(i, t)| resolve_informational_field(i, t)) .collect::>>()?; + // Even with collecting results into a set, drop duplicates early + arg_fields.sort(); + arg_fields.dedup(); let return_type = udwf .field(WindowUDFFieldArgs::new(&arg_fields, udwf.name())) .map(|f| { diff --git a/datafusion/common/src/types/native.rs b/datafusion/common/src/types/native.rs index 580d572af4c0f..ff5a04f3b2390 100644 --- a/datafusion/common/src/types/native.rs +++ b/datafusion/common/src/types/native.rs @@ -23,7 +23,7 @@ use crate::error::{_internal_err, Result}; use arrow::compute::can_cast_types; use arrow::datatypes::{ DECIMAL32_MAX_PRECISION, DECIMAL64_MAX_PRECISION, DECIMAL128_MAX_PRECISION, DataType, - Field, FieldRef, Fields, IntervalUnit, TimeUnit, UnionFields, + Field, FieldRef, Fields, IntervalUnit, TimeUnit, UnionFields, UnionMode, }; use std::{fmt::Display, sync::Arc}; @@ -432,6 +432,23 @@ impl LogicalType for NativeType { *mode, ) } + // There is no mode to preserve with null origin, so pick dense encoding + (Self::Union(to_fields), Null) => Union( + to_fields + .iter() + .map(|(type_id, field)| { + Ok(( + *type_id, + Arc::new(Field::new( + field.name.clone(), + field.logical_type.default_cast_for(&Null)?, + field.nullable, + )), + )) + }) + .collect::>()?, + UnionMode::Dense, + ), _ => { return _internal_err!( "Unavailable default cast for native type {} from physical type {}", @@ -645,4 +662,109 @@ mod tests { )))); assert_snapshot!(map, @"Map(non-null String)"); } + + #[test] + fn test_default_cast_for_null_origin() { + // Native types with several physical types: pick a canonical one + assert_eq!( + NativeType::String.default_cast_for(&DataType::Null).unwrap(), + DataType::Utf8View + ); + assert_eq!( + NativeType::Binary.default_cast_for(&DataType::Null).unwrap(), + DataType::BinaryView + ); + assert_eq!( + NativeType::Date.default_cast_for(&DataType::Null).unwrap(), + DataType::Date32 + ); + + // Decimals pick the narrowest physical type holding the precision + assert_eq!( + NativeType::Decimal(9, 2) + .default_cast_for(&DataType::Null) + .unwrap(), + DataType::Decimal32(9, 2) + ); + assert_eq!( + NativeType::Decimal(18, 2) + .default_cast_for(&DataType::Null) + .unwrap(), + DataType::Decimal64(18, 2) + ); + assert_eq!( + NativeType::Decimal(38, 10) + .default_cast_for(&DataType::Null) + .unwrap(), + DataType::Decimal128(38, 10) + ); + assert_eq!( + NativeType::Decimal(50, 2) + .default_cast_for(&DataType::Null) + .unwrap(), + DataType::Decimal256(50, 2) + ); + + // nested types + let list = NativeType::List(Arc::new(LogicalField::from(&Field::new( + "item", + DataType::Int32, + true, + )))); + assert_eq!( + list.default_cast_for(&DataType::Null).unwrap(), + DataType::List(Arc::new(Field::new("item", DataType::Int32, true))) + ); + + let struct_type = NativeType::Struct(LogicalFields::from(&Fields::from(vec![ + Field::new("name", DataType::Utf8, false), + Field::new("age", DataType::Int32, true), + ]))); + assert_eq!( + struct_type.default_cast_for(&DataType::Null).unwrap(), + DataType::Struct(Fields::from(vec![ + // The logical `String` field resolves to its canonical type + Field::new("name", DataType::Utf8View, false), + Field::new("age", DataType::Int32, true), + ])) + ); + } + + #[test] + fn test_default_cast_for_union() { + let origin_fields = UnionFields::try_new( + vec![3, 7], + vec![ + Field::new("a", DataType::Int32, true), + Field::new("b", DataType::Utf8, false), + ], + ) + .unwrap(); + let union_type = NativeType::Union(LogicalUnionFields::from(&origin_fields)); + + // From a `Null` origin there are only type ids, but no mode or string + let expected_fields = UnionFields::try_new( + vec![3, 7], + vec![ + Field::new("a", DataType::Int32, true), + Field::new("b", DataType::Utf8View, false), + ], + ) + .unwrap(); + assert_eq!( + union_type.default_cast_for(&DataType::Null).unwrap(), + DataType::Union(expected_fields, UnionMode::Dense) + ); + + // From a union origin both the mode and the encodings are preserved + assert_eq!( + union_type + .default_cast_for(&DataType::Union( + origin_fields.clone(), + UnionMode::Sparse + )) + .unwrap(), + DataType::Union(origin_fields, UnionMode::Sparse) + ); + } } diff --git a/datafusion/expr-common/src/signature.rs b/datafusion/expr-common/src/signature.rs index 60591669714d4..083f9518cb7c2 100644 --- a/datafusion/expr-common/src/signature.rs +++ b/datafusion/expr-common/src/signature.rs @@ -22,9 +22,8 @@ use std::hash::Hash; use std::sync::Arc; use arrow::datatypes::{ - DECIMAL_DEFAULT_SCALE, DECIMAL32_MAX_PRECISION, DECIMAL64_MAX_PRECISION, - DECIMAL128_MAX_PRECISION, DataType, Decimal128Type, DecimalType, Field, IntervalUnit, - TimeUnit, + DECIMAL32_MAX_PRECISION, DECIMAL64_MAX_PRECISION, DECIMAL128_MAX_PRECISION, DataType, + Decimal128Type, DecimalType, Field, IntervalUnit, TimeUnit, }; use datafusion_common::types::{LogicalType, LogicalTypeRef, NativeType}; use datafusion_common::utils::ListCoercion; @@ -417,12 +416,10 @@ impl TypeSignatureClass { /// /// This is used for `information_schema` and can be used to generate /// documentation or error messages. - // Remove with `get_example_types` - #[deprecated(since = "53.0.0", note = "See get_representative_types instead")] + // Deprecated, remove with `get_example_types` fn get_example_types(&self) -> Vec { match self { TypeSignatureClass::Any => vec![], - #[expect(deprecated)] TypeSignatureClass::Native(l) => get_data_types(l.native()), TypeSignatureClass::Timestamp => { vec![ @@ -490,10 +487,14 @@ impl TypeSignatureClass { TypeSignatureClass::Binary => { vec![NativeType::Binary] } - TypeSignatureClass::Decimal => vec![NATIVE_TYPE_DECIMAL], + TypeSignatureClass::Decimal => vec![Decimal128Type::DEFAULT_TYPE.into()], TypeSignatureClass::Float => vec![NativeType::Float64], TypeSignatureClass::Numeric => { - vec![NativeType::Float64, NativeType::Int64, NATIVE_TYPE_DECIMAL] + vec![ + NativeType::Float64, + NativeType::Int64, + Decimal128Type::DEFAULT_TYPE.into(), + ] } } } @@ -637,28 +638,7 @@ impl Display for ArrayFunctionArgument { } } -/// Constant that is used as a Decimal type for `get_example_types` -/// Use Decimal128 precision as a reasonable default -const NATIVE_TYPE_DECIMAL: NativeType = - NativeType::Decimal(DECIMAL128_MAX_PRECISION, DECIMAL_DEFAULT_SCALE); - -/// Native types for `get_example_types` -static EXAMPLE_NUMERIC_TYPES: &[NativeType] = &[ - NativeType::Int8, - NativeType::Int16, - NativeType::Int32, - NativeType::Int64, - NativeType::UInt8, - NativeType::UInt16, - NativeType::UInt32, - NativeType::UInt64, - NativeType::Float16, - NativeType::Float32, - NativeType::Float64, - NATIVE_TYPE_DECIMAL, -]; - -#[deprecated(since = "53.0.0", note = "See get_representative_types instead")] +// Deprecated, remove with `get_example_types` static NUMERICS: &[DataType] = &[ DataType::Int8, DataType::Int16, @@ -950,9 +930,8 @@ impl TypeSignature { /// /// This is used for `information_schema` and can be used to generate /// documentation or error messages. - #[deprecated(since = "53.0.0", note = "See get_representative_types instead")] + #[deprecated(since = "55.0.0", note = "See get_representative_types instead")] pub fn get_example_types(&self) -> Vec> { - #[expect(deprecated)] match self { TypeSignature::Exact(types) => vec![types.clone()], TypeSignature::OneOf(types) => types @@ -1007,12 +986,13 @@ impl TypeSignature { } } - /// Return example acceptable types for this `TypeSignature`' + /// Return example acceptable types for this `TypeSignature' /// /// Returns a `Vec` for each argument to the function /// /// This is used for `information_schema` and can be used to generate - /// documentation or error messages. + /// documentation or error messages. The returned types are the types a caller may pass. + /// Callers computing return types must apply coercion themselves. pub fn get_representative_types(&self) -> Vec> { match self { TypeSignature::Exact(types) => vec![ @@ -1057,10 +1037,10 @@ impl TypeSignature { .cloned() .map(|data_type| vec![data_type.into()]) .collect(), - TypeSignature::Numeric(arg_count) => EXAMPLE_NUMERIC_TYPES - .iter() - .cloned() - .map(|numeric_type| vec![numeric_type; *arg_count]) + TypeSignature::Numeric(arg_count) => TypeSignatureClass::Numeric + .get_representative_types() + .into_iter() + .map(|t| vec![t; *arg_count]) .collect(), TypeSignature::String(arg_count) => { vec![vec![NativeType::String; *arg_count]] @@ -1076,7 +1056,7 @@ impl TypeSignature { } } -#[deprecated(since = "53.0.0", note = "See get_representative_types instead")] +// Deprecated, remove with get_data_types fn get_data_types(native_type: &NativeType) -> Vec { match native_type { NativeType::Null => vec![DataType::Null], @@ -1948,18 +1928,12 @@ mod tests { assert_eq!( types, vec![ - vec![NativeType::Int8, NativeType::Int8], - vec![NativeType::Int16, NativeType::Int16], - vec![NativeType::Int32, NativeType::Int32], - vec![NativeType::Int64, NativeType::Int64], - vec![NativeType::UInt8, NativeType::UInt8], - vec![NativeType::UInt16, NativeType::UInt16], - vec![NativeType::UInt32, NativeType::UInt32], - vec![NativeType::UInt64, NativeType::UInt64], - vec![NativeType::Float16, NativeType::Float16], - vec![NativeType::Float32, NativeType::Float32], vec![NativeType::Float64, NativeType::Float64], - vec![NATIVE_TYPE_DECIMAL, NATIVE_TYPE_DECIMAL], + vec![NativeType::Int64, NativeType::Int64], + vec![ + Decimal128Type::DEFAULT_TYPE.into(), + Decimal128Type::DEFAULT_TYPE.into() + ], ] ); diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 573fb04b3451b..976b98072fcfa 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -899,6 +899,57 @@ date_trunc Time(ns) [precision, expression] [String, Time(ns)] SCALAR Truncates date_trunc Timestamp(ns) [precision, expression] [String, Timestamp(ns)] SCALAR Truncates a timestamp or time value to a specified precision. date_trunc(precision, expression) date_trunc Timestamp(ns, "+TZ") [precision, expression] [String, Timestamp(ns, "+TZ")] SCALAR Truncates a timestamp or time value to a specified precision. date_trunc(precision, expression) +# Schema for function with TypeSignature::Numeric +query TTITI rowsort +select specific_name, data_type, ordinal_position, parameter_mode, rid from information_schema.parameters where specific_name = 'abs'; +---- +abs Decimal(38, 10) 1 IN 0 +abs Decimal(38, 10) 1 OUT 0 +abs Float64 1 IN 1 +abs Float64 1 OUT 1 +abs Int64 1 IN 2 +abs Int64 1 OUT 2 + +query TTT rowsort +select routine_name, data_type, function_type from information_schema.routines where routine_name = 'abs'; +---- +abs Decimal(38, 10) SCALAR +abs Float64 SCALAR +abs Int64 SCALAR + +# Schema for function with TypeSignature::Coercible plus uniform with exact float types +query TTITI rowsort +select specific_name, data_type, ordinal_position, parameter_mode, rid from information_schema.parameters where specific_name = 'floor'; +---- +floor Decimal(38, 10) 1 IN 0 +floor Decimal(38, 10) 1 OUT 0 +floor Float32 1 IN 1 +floor Float32 1 OUT 1 +floor Float64 1 IN 2 +floor Float64 1 OUT 2 + +query TTT rowsort +select routine_name, data_type, function_type from information_schema.routines where routine_name = 'floor'; +---- +floor Decimal(38, 10) SCALAR +floor Float32 SCALAR +floor Float64 SCALAR + +# Schemas for date/duration - exact type for Date32/Date64 (logical type is provided) +query TTITI rowsort +select specific_name, data_type, ordinal_position, parameter_mode, rid from information_schema.parameters where specific_name = 'to_char' and (data_type like '%Date%' or data_type like '%Duration%'); +---- +to_char Date 1 IN 0 +to_char Duration(ms) 1 IN 1 +to_char Duration(ns) 1 IN 2 +to_char Duration(s) 1 IN 3 +to_char Duration(µs) 1 IN 4 + +query TTT rowsort +select routine_name, data_type, function_type from information_schema.routines where routine_name = 'to_char'; +---- +to_char String SCALAR + # Table functions (UDTFs) appear in information_schema.routines with # function_type = TABLE and data_type = TABLE. # Note: built-in `generate_series` and `range` are registered as BOTH a From 3d1b43a29535b5ae42adf6e2b59a6edfbd6e78b5 Mon Sep 17 00:00:00 2001 From: theirix Date: Tue, 18 Aug 2026 14:35:07 +0100 Subject: [PATCH 09/19] Format --- datafusion/common/src/types/native.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/datafusion/common/src/types/native.rs b/datafusion/common/src/types/native.rs index ff5a04f3b2390..19278d04e3f09 100644 --- a/datafusion/common/src/types/native.rs +++ b/datafusion/common/src/types/native.rs @@ -667,11 +667,15 @@ mod tests { fn test_default_cast_for_null_origin() { // Native types with several physical types: pick a canonical one assert_eq!( - NativeType::String.default_cast_for(&DataType::Null).unwrap(), + NativeType::String + .default_cast_for(&DataType::Null) + .unwrap(), DataType::Utf8View ); assert_eq!( - NativeType::Binary.default_cast_for(&DataType::Null).unwrap(), + NativeType::Binary + .default_cast_for(&DataType::Null) + .unwrap(), DataType::BinaryView ); assert_eq!( From 6997029db08d07dc5887b4e9377d356a3d9e2dec Mon Sep 17 00:00:00 2001 From: theirix Date: Sun, 27 Sep 2026 22:12:57 +0100 Subject: [PATCH 10/19] Sync test cases with main --- datafusion/sqllogictest/test_files/information_schema.slt | 4 ++-- parquet-testing | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index c1ead739c1c25..2fdf082235ac3 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -922,8 +922,8 @@ abs Int64 SCALAR query TTITI rowsort select specific_name, data_type, ordinal_position, parameter_mode, rid from information_schema.parameters where specific_name = 'floor'; ---- +floor Decimal(29, 0) 1 OUT 0 floor Decimal(38, 10) 1 IN 0 -floor Decimal(38, 10) 1 OUT 0 floor Float32 1 IN 1 floor Float32 1 OUT 1 floor Float64 1 IN 2 @@ -932,7 +932,7 @@ floor Float64 1 OUT 2 query TTT rowsort select routine_name, data_type, function_type from information_schema.routines where routine_name = 'floor'; ---- -floor Decimal(38, 10) SCALAR +floor Decimal(29, 0) SCALAR floor Float32 SCALAR floor Float64 SCALAR diff --git a/parquet-testing b/parquet-testing index 107b36603e051..56653c437c809 160000 --- a/parquet-testing +++ b/parquet-testing @@ -1 +1 @@ -Subproject commit 107b36603e051aee26bd93e04b871034f6c756c0 +Subproject commit 56653c437c8092f704a092d0d1d4e600124cd49f From 7fff35aac4f5273473d6c46fb95c4bf71256e452 Mon Sep 17 00:00:00 2001 From: theirix Date: Sun, 27 Sep 2026 22:17:11 +0100 Subject: [PATCH 11/19] Avoid dedup with arg sorting --- datafusion/catalog/src/information_schema.rs | 70 ++++++++++++++++---- 1 file changed, 58 insertions(+), 12 deletions(-) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index d0a29d513f7d6..7df872c41ad45 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -475,14 +475,11 @@ fn get_udf_args_and_return_types( arg_types .into_iter() .map(|arg_types| { - let mut arg_fields = arg_types + let arg_fields = arg_types .iter() .enumerate() .map(|(i, t)| resolve_informational_field(i, t)) .collect::>>()?; - // Even with collecting results into a set, drop duplicates early - arg_fields.sort(); - arg_fields.dedup(); let scalar_arguments = vec![None; arg_fields.len()]; let return_type = udf .return_field_from_args(ReturnFieldArgs { @@ -516,14 +513,11 @@ fn get_udaf_args_and_return_types( arg_types .into_iter() .map(|arg_types| { - let mut arg_fields = arg_types + let arg_fields = arg_types .iter() .enumerate() .map(|(i, t)| resolve_informational_field(i, t)) .collect::>>()?; - // Even with collecting results into a set, drop duplicates early - arg_fields.sort(); - arg_fields.dedup(); let return_type = udaf .return_field(&arg_fields) .map(|f| { @@ -553,14 +547,11 @@ fn get_udwf_args_and_return_types( arg_types .into_iter() .map(|arg_types| { - let mut arg_fields = arg_types + let arg_fields = arg_types .iter() .enumerate() .map(|(i, t)| resolve_informational_field(i, t)) .collect::>>()?; - // Even with collecting results into a set, drop duplicates early - arg_fields.sort(); - arg_fields.dedup(); let return_type = udwf .field(WindowUDFFieldArgs::new(&arg_fields, udwf.name())) .map(|f| { @@ -1521,6 +1512,8 @@ mod tests { use super::*; use crate::CatalogProvider; use arrow::array::Array; + use datafusion_common::ScalarValue; + use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl}; #[test] fn schemata_builder_emits_canonical_schema_and_rows() { @@ -1591,6 +1584,59 @@ mod tests { assert_eq!("BASE TABLE", builder.table_types.finish().value(0)); } + // UDF + #[derive(Debug, PartialEq, Eq, Hash)] + struct TestScalarUDF { + signature: Signature, + } + impl ScalarUDFImpl for TestScalarUDF { + fn name(&self) -> &str { + "TestScalarUDF" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> Result { + Ok(arg_types.last().unwrap().clone()) + } + + fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result { + Ok(ColumnarValue::Scalar(ScalarValue::from("a"))) + } + } + + #[test] + fn test_get_udf_args_and_return_types() -> Result<()> { + // heterogeneous arguments to test mixed arguments retrieval + let signature = Signature::exact( + [ + vec![DataType::Int32; 6], + vec![DataType::Float32; 6], + vec![DataType::Utf8; 1], + ] + .concat(), + Volatility::Stable, + ); + let udf = Arc::new(ScalarUDF::from(TestScalarUDF { signature })); + let result = get_udf_args_and_return_types(&udf)?; + assert_eq!(result.len(), 1); + let (args, ret) = result.iter().next().unwrap(); + assert_eq!( + *args, + [ + vec![String::from("Int32"); 6], + vec![String::from("Float32"); 6], + vec![String::from("String"); 1] + ] + .concat() + ); + assert_eq!(*ret, Some(String::from("String"))); + + Ok(()) + } + #[derive(Debug)] struct Fixture; From 736e56d62e107a2e616b28caa9e72ce1a0669d7d Mon Sep 17 00:00:00 2001 From: theirix Date: Sun, 4 Oct 2026 22:14:24 +0100 Subject: [PATCH 12/19] Resolve ambigious native-to-physical types --- datafusion/catalog/src/information_schema.rs | 183 ++++++++----------- 1 file changed, 81 insertions(+), 102 deletions(-) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index 7df872c41ad45..a911d7459749b 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -43,6 +43,7 @@ use datafusion_expr::{TableType, Volatility}; use datafusion_physical_plan::SendableRecordBatchStream; use datafusion_physical_plan::stream::RecordBatchStreamAdapter; use datafusion_physical_plan::streaming::PartitionStream; +use itertools::Itertools; use std::collections::{BTreeSet, HashMap, HashSet}; use std::fmt::Debug; use std::sync::Arc; @@ -454,120 +455,98 @@ impl InformationSchemaConfig { } } -/// Build the argument field for `information_schema` to provide a return type -fn resolve_informational_field(idx: usize, t: &NativeType) -> Result { - // Since a native type maps to several physical types, resolve it against `Null` data type - // to get the canonical `DataType` for the native type - let data_type = t.default_cast_for(&DataType::Null)?; - Ok(Arc::new(Field::new(format!("arg_{idx}"), data_type, true))) +/// Origins used to enumerate the physical types a native type can take +const RESOLVE_CAST_SOURCES: [DataType; 2] = [DataType::Null, DataType::LargeUtf8]; + +/// Build argument fields for `information_schema` to provide possbile return types +fn resolve_informational_fields(idx: usize, t: &NativeType) -> Result> { + // Since native types map to several physical types, resolve it against + // ambigious types to get canonical `DataType`s for the native type + let data_types = RESOLVE_CAST_SOURCES + .iter() + .map(|source| t.default_cast_for(source)) + .collect::, _>>()?; + Ok(data_types + .into_iter() + .unique() + .map(|dt| Arc::new(Field::new(format!("arg_{idx}"), dt, true))) + .collect()) } -/// get the arguments and return types of a UDF -/// returns a tuple of (arg_types, return_type) -fn get_udf_args_and_return_types( - udf: &Arc, -) -> Result, Option)>> { - let signature = udf.signature(); +/// Function information schema is a set of tuples - argument types and an optional return type +type FunctionInformationSchema = BTreeSet<(Vec, Option)>; + +/// Get the arguments and return types of a function from its signature +fn get_args_and_return_types( + signature: &Signature, + return_field: impl Fn(&[FieldRef]) -> Result, +) -> Result { let arg_types = signature.type_signature.get_representative_types(); if arg_types.is_empty() { - Ok(vec![(vec![], None)].into_iter().collect::>()) - } else { - arg_types - .into_iter() - .map(|arg_types| { - let arg_fields = arg_types - .iter() - .enumerate() - .map(|(i, t)| resolve_informational_field(i, t)) - .collect::>>()?; - let scalar_arguments = vec![None; arg_fields.len()]; - let return_type = udf - .return_field_from_args(ReturnFieldArgs { - arg_fields: &arg_fields, - scalar_arguments: &scalar_arguments, - }) - .map(|f| { - remove_native_type_prefix(&NativeType::from( - f.data_type().clone(), - )) - }) - .ok(); - let arg_types = arg_types - .into_iter() - .map(|t| remove_native_type_prefix(&t)) - .collect::>(); - Ok((arg_types, return_type)) - }) - .collect::>>() - } + // Edge case if function doesn't have arguments + return Ok(BTreeSet::from([(vec![], None)])); + } + arg_types + .into_iter() + .map(|arg_types| { + // Get possible types for each input arg + let arg_fields = arg_types + .iter() + .enumerate() + .map(|(i, t)| resolve_informational_fields(i, t)) + .collect::>>()?; + // Build combinations of arg types with the return type + let return_types = arg_fields + .into_iter() + .multi_cartesian_product() + .filter_map(|arg_fields| return_field(&arg_fields).ok()) + .map(|f| Some(remove_native_type_prefix(&f.data_type().into()))) + .collect::>(); + let return_types = if return_types.is_empty() { + // Indicate `None` if the return type cannot be represented from a signature, + BTreeSet::from([None]) + } else { + return_types + }; + let arg_types = arg_types + .iter() + .map(remove_native_type_prefix) + .collect::>(); + let tuples = return_types + .into_iter() + .map(move |return_type| (arg_types.clone(), return_type)); + Ok(tuples) + }) + .flatten_ok() + .collect() +} + +fn get_udf_args_and_return_types( + udf: &Arc, +) -> Result { + get_args_and_return_types(udf.signature(), |arg_fields| { + let scalar_arguments = &vec![None; arg_fields.len()]; + udf.return_field_from_args(ReturnFieldArgs { + arg_fields, + scalar_arguments, + }) + }) } fn get_udaf_args_and_return_types( udaf: &Arc, -) -> Result, Option)>> { - let signature = udaf.signature(); - let arg_types = signature.type_signature.get_representative_types(); - if arg_types.is_empty() { - Ok(vec![(vec![], None)].into_iter().collect::>()) - } else { - arg_types - .into_iter() - .map(|arg_types| { - let arg_fields = arg_types - .iter() - .enumerate() - .map(|(i, t)| resolve_informational_field(i, t)) - .collect::>>()?; - let return_type = udaf - .return_field(&arg_fields) - .map(|f| { - remove_native_type_prefix(&NativeType::from( - f.data_type().clone(), - )) - }) - .ok(); - let arg_types = arg_types - .into_iter() - .map(|t| remove_native_type_prefix(&t)) - .collect::>(); - Ok((arg_types, return_type)) - }) - .collect::>>() - } +) -> Result { + get_args_and_return_types(udaf.signature(), |arg_fields| { + udaf.return_field(arg_fields) + }) } fn get_udwf_args_and_return_types( udwf: &Arc, -) -> Result, Option)>> { - let signature = udwf.signature(); - let arg_types = signature.type_signature.get_representative_types(); - if arg_types.is_empty() { - Ok(vec![(vec![], None)].into_iter().collect::>()) - } else { - arg_types - .into_iter() - .map(|arg_types| { - let arg_fields = arg_types - .iter() - .enumerate() - .map(|(i, t)| resolve_informational_field(i, t)) - .collect::>>()?; - let return_type = udwf - .field(WindowUDFFieldArgs::new(&arg_fields, udwf.name())) - .map(|f| { - remove_native_type_prefix(&NativeType::from( - f.data_type().clone(), - )) - }) - .ok(); - let arg_types = arg_types - .into_iter() - .map(|t| remove_native_type_prefix(&t)) - .collect::>(); - Ok((arg_types, return_type)) - }) - .collect::>>() - } +) -> Result { + get_args_and_return_types(udwf.signature(), |arg_fields| { + udwf.field(WindowUDFFieldArgs::new(arg_fields, udwf.name())) + }) } #[inline] From 082df116e91b369e55f761bdd786da5aa9aef358 Mon Sep 17 00:00:00 2001 From: theirix Date: Sun, 4 Oct 2026 22:14:45 +0100 Subject: [PATCH 13/19] Test ambigious cases --- .../test_files/information_schema.slt | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 2fdf082235ac3..0a02db99b36fc 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -918,6 +918,12 @@ abs Decimal(38, 10) SCALAR abs Float64 SCALAR abs Int64 SCALAR +query TT rowsort +select routine_name, data_type from information_schema.routines where routine_name = 'length'; +---- +length Int32 +length Int64 + # Schema for function with TypeSignature::Coercible plus uniform with exact float types query TTITI rowsort select specific_name, data_type, ordinal_position, parameter_mode, rid from information_schema.parameters where specific_name = 'floor'; @@ -951,6 +957,23 @@ select routine_name, data_type, function_type from information_schema.routines w ---- to_char String SCALAR +# Return type cannot be deduced from the signature alone +query TTT +select routine_name, data_type, function_type from information_schema.routines where routine_name = 'date_part'; +---- +date_part NULL SCALAR + +# No args +query TTT +select routine_name, data_type, function_type from information_schema.routines where routine_name = 'pi'; +---- +pi NULL SCALAR + +query I +select count(*) from information_schema.parameters where specific_name = 'date_part' and parameter_mode = 'OUT'; +---- +0 + # Table functions (UDTFs) appear in information_schema.routines with # function_type = TABLE and data_type = TABLE. # Note: built-in `generate_series` and `range` are registered as BOTH a From 0c8e779281ada7442e4fa8ff54335abb0893e068 Mon Sep 17 00:00:00 2001 From: theirix Date: Sun, 4 Oct 2026 22:15:40 +0100 Subject: [PATCH 14/19] Avoid committing parquet-testing --- parquet-testing | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parquet-testing b/parquet-testing index 56653c437c809..107b36603e051 160000 --- a/parquet-testing +++ b/parquet-testing @@ -1 +1 @@ -Subproject commit 56653c437c8092f704a092d0d1d4e600124cd49f +Subproject commit 107b36603e051aee26bd93e04b871034f6c756c0 From f8a88d91f9cd872df1ffb45a45419b6415259805 Mon Sep 17 00:00:00 2001 From: theirix Date: Sun, 4 Oct 2026 22:20:34 +0100 Subject: [PATCH 15/19] Typos --- datafusion/catalog/src/information_schema.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index a911d7459749b..b275b89168aa7 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -458,10 +458,10 @@ impl InformationSchemaConfig { /// Origins used to enumerate the physical types a native type can take const RESOLVE_CAST_SOURCES: [DataType; 2] = [DataType::Null, DataType::LargeUtf8]; -/// Build argument fields for `information_schema` to provide possbile return types +/// Build argument fields for `information_schema` to provide possible return types fn resolve_informational_fields(idx: usize, t: &NativeType) -> Result> { // Since native types map to several physical types, resolve it against - // ambigious types to get canonical `DataType`s for the native type + // ambiguous types to get canonical `DataType`s for the native type let data_types = RESOLVE_CAST_SOURCES .iter() .map(|source| t.default_cast_for(source)) From b7236957bdc49491d508f7af20bc39f635e854f0 Mon Sep 17 00:00:00 2001 From: theirix Date: Mon, 5 Oct 2026 19:26:48 +0100 Subject: [PATCH 16/19] Handle types missing from default_cast_for --- datafusion/catalog/src/information_schema.rs | 28 +++++++++++++------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index b275b89168aa7..1163770989c3b 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -459,18 +459,16 @@ impl InformationSchemaConfig { const RESOLVE_CAST_SOURCES: [DataType; 2] = [DataType::Null, DataType::LargeUtf8]; /// Build argument fields for `information_schema` to provide possible return types -fn resolve_informational_fields(idx: usize, t: &NativeType) -> Result> { +fn resolve_informational_fields(idx: usize, t: &NativeType) -> Vec { // Since native types map to several physical types, resolve it against - // ambiguous types to get canonical `DataType`s for the native type - let data_types = RESOLVE_CAST_SOURCES + // ambiguous types to get canonical `DataType`s for the native type. + // Skip origins the type has no cast from (e.g. `Struct` from `LargeUtf8`) + RESOLVE_CAST_SOURCES .iter() - .map(|source| t.default_cast_for(source)) - .collect::, _>>()?; - Ok(data_types - .into_iter() + .filter_map(|source| t.default_cast_for(source).ok()) .unique() .map(|dt| Arc::new(Field::new(format!("arg_{idx}"), dt, true))) - .collect()) + .collect() } /// Function information schema is a set of tuples - argument types and an optional return type @@ -494,7 +492,7 @@ fn get_args_and_return_types( .iter() .enumerate() .map(|(i, t)| resolve_informational_fields(i, t)) - .collect::>>()?; + .collect::>(); // Build combinations of arg types with the return type let return_types = arg_fields .into_iter() @@ -1491,6 +1489,7 @@ mod tests { use super::*; use crate::CatalogProvider; use arrow::array::Array; + use arrow::datatypes::Fields; use datafusion_common::ScalarValue; use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl}; @@ -1616,6 +1615,17 @@ mod tests { Ok(()) } + #[test] + fn test_get_udf_args_and_return_types_nested() -> Result<()> { + let struct_type = + DataType::Struct(Fields::from(vec![Field::new("a", DataType::Int32, true)])); + let signature = Signature::exact(vec![struct_type], Volatility::Stable); + let udf = Arc::new(ScalarUDF::from(TestScalarUDF { signature })); + let result = get_udf_args_and_return_types(&udf)?; + assert_eq!(result.len(), 1); + Ok(()) + } + #[derive(Debug)] struct Fixture; From 305c36d2f3ff4b8504bdacbca8f78d91ffb76869 Mon Sep 17 00:00:00 2001 From: theirix Date: Mon, 5 Oct 2026 21:50:54 +0100 Subject: [PATCH 17/19] Coerce deduced return types --- datafusion/catalog/src/information_schema.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index 1163770989c3b..85222bc2f6ec5 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -36,6 +36,7 @@ use datafusion_common::types::{LogicalType, NativeType}; use datafusion_execution::TaskContext; use datafusion_execution::runtime_env::RuntimeEnv; use datafusion_expr::function::WindowUDFFieldArgs; +use datafusion_expr::type_coercion::functions::fields_with_udf; use datafusion_expr::{ AggregateUDF, ReturnFieldArgs, ScalarUDF, Signature, TypeSignature, WindowUDF, }; @@ -523,6 +524,7 @@ fn get_udf_args_and_return_types( udf: &Arc, ) -> Result { get_args_and_return_types(udf.signature(), |arg_fields| { + let arg_fields = &fields_with_udf(arg_fields, udf.as_ref())?; let scalar_arguments = &vec![None; arg_fields.len()]; udf.return_field_from_args(ReturnFieldArgs { arg_fields, @@ -535,6 +537,7 @@ fn get_udaf_args_and_return_types( udaf: &Arc, ) -> Result { get_args_and_return_types(udaf.signature(), |arg_fields| { + let arg_fields = &fields_with_udf(arg_fields, udaf.as_ref())?; udaf.return_field(arg_fields) }) } @@ -543,6 +546,7 @@ fn get_udwf_args_and_return_types( udwf: &Arc, ) -> Result { get_args_and_return_types(udwf.signature(), |arg_fields| { + let arg_fields = &fields_with_udf(arg_fields, udwf.as_ref())?; udwf.field(WindowUDFFieldArgs::new(arg_fields, udwf.name())) }) } From af0662a8568c698acc7bc7bf170ad29a51548f4d Mon Sep 17 00:00:00 2001 From: theirix Date: Mon, 5 Oct 2026 21:51:33 +0100 Subject: [PATCH 18/19] SLT: date_trunc announces Timestamp return type --- .../sqllogictest/test_files/information_schema.slt | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 0a02db99b36fc..0907b857673ba 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -844,8 +844,6 @@ string_agg String AGGREGATE query TTTTTTTBTTTT rowsort select * from information_schema.routines where routine_name = 'date_trunc' OR routine_name = 'string_agg' OR routine_name = 'rank' ORDER BY routine_name ---- -datafusion public date_trunc datafusion public date_trunc FUNCTION true Date SCALAR Truncates a timestamp or time value to a specified precision. date_trunc(precision, expression) -datafusion public date_trunc datafusion public date_trunc FUNCTION true String SCALAR Truncates a timestamp or time value to a specified precision. date_trunc(precision, expression) datafusion public date_trunc datafusion public date_trunc FUNCTION true Time(ns) SCALAR Truncates a timestamp or time value to a specified precision. date_trunc(precision, expression) datafusion public date_trunc datafusion public date_trunc FUNCTION true Timestamp(ns) SCALAR Truncates a timestamp or time value to a specified precision. date_trunc(precision, expression) datafusion public date_trunc datafusion public date_trunc FUNCTION true Timestamp(ns, "+TZ") SCALAR Truncates a timestamp or time value to a specified precision. date_trunc(precision, expression) @@ -861,12 +859,12 @@ false query TTTITTTTBI select * from information_schema.parameters where specific_name = 'date_trunc' OR specific_name = 'string_agg' OR specific_name = 'rank' ORDER BY specific_name, rid, data_type; ---- -datafusion public date_trunc 1 OUT NULL Date NULL false 0 datafusion public date_trunc 2 IN expression Date NULL false 0 datafusion public date_trunc 1 IN precision String NULL false 0 +datafusion public date_trunc 1 OUT NULL Timestamp(ns) NULL false 0 datafusion public date_trunc 1 IN precision String NULL false 1 datafusion public date_trunc 2 IN expression String NULL false 1 -datafusion public date_trunc 1 OUT NULL String NULL false 1 +datafusion public date_trunc 1 OUT NULL Timestamp(ns) NULL false 1 datafusion public date_trunc 1 IN precision String NULL false 2 datafusion public date_trunc 2 IN expression Time(ns) NULL false 2 datafusion public date_trunc 1 OUT NULL Time(ns) NULL false 2 @@ -894,9 +892,9 @@ repeat String 1 OUT 0 query TT??TTT rowsort show functions like 'date_trunc'; ---- -date_trunc Date [precision, expression] [String, Date] SCALAR Truncates a timestamp or time value to a specified precision. date_trunc(precision, expression) -date_trunc String [precision, expression] [String, String] SCALAR Truncates a timestamp or time value to a specified precision. date_trunc(precision, expression) date_trunc Time(ns) [precision, expression] [String, Time(ns)] SCALAR Truncates a timestamp or time value to a specified precision. date_trunc(precision, expression) +date_trunc Timestamp(ns) [precision, expression] [String, Date] SCALAR Truncates a timestamp or time value to a specified precision. date_trunc(precision, expression) +date_trunc Timestamp(ns) [precision, expression] [String, String] SCALAR Truncates a timestamp or time value to a specified precision. date_trunc(precision, expression) date_trunc Timestamp(ns) [precision, expression] [String, Timestamp(ns)] SCALAR Truncates a timestamp or time value to a specified precision. date_trunc(precision, expression) date_trunc Timestamp(ns, "+TZ") [precision, expression] [String, Timestamp(ns, "+TZ")] SCALAR Truncates a timestamp or time value to a specified precision. date_trunc(precision, expression) From 91efce9dc48130cb21575ebaacdb37f56fa90a19 Mon Sep 17 00:00:00 2001 From: theirix Date: Mon, 5 Oct 2026 21:51:49 +0100 Subject: [PATCH 19/19] SLT: verify generate_series returning Date --- .../sqllogictest/test_files/information_schema.slt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 0907b857673ba..27bc27915a523 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -972,6 +972,18 @@ select count(*) from information_schema.parameters where specific_name = 'date_p ---- 0 +# Verify coerced return type of `generate_series`: it should be Date32 +query TI?T rowsort +select specific_name, rid, + array_agg(data_type order by ordinal_position) filter (where parameter_mode = 'IN') as args, + max(data_type) filter (where parameter_mode = 'OUT') as return +from information_schema.parameters where specific_name = 'generate_series' +group by specific_name, rid +having args[1] = 'String' and args[2] like 'Date' and args[3] like 'Interval%'; +---- +generate_series 7 [String, Date, Interval(DayTime)] List(Date) +generate_series 8 [String, Date, Interval(MonthDayNano)] List(Date) + # Table functions (UDTFs) appear in information_schema.routines with # function_type = TABLE and data_type = TABLE. # Note: built-in `generate_series` and `range` are registered as BOTH a