From d4049bf3f69e7f5e2a859838814a1e6a53c343ae Mon Sep 17 00:00:00 2001 From: Christian McArthur Date: Tue, 26 May 2026 11:08:11 -0400 Subject: [PATCH 1/4] feat: add array_sum scalar function Adds `array_sum(array)` returning the sum of elements in a numeric array. Aliased as `list_sum`. Part of the per-function split sequence on tracking issue #21536, following the pattern of the already-merged PRs in this series (cosine_distance #21542, inner_product #21861, array_normalize #22013, array_scale #22466). Semantics: - NULL row in array -> NULL row out - NULL elements are skipped (SQL aggregate convention; matches PostgreSQL array_sum, DuckDB list_sum, Spark aggregate). A row whose every element is NULL yields NULL. - Empty array -> 0.0 (additive identity, matches SQL SUM over no rows conceptually, and DuckDB list_sum([]) = 0) Input is List/LargeList/FixedSizeList of any numeric type; elements are coerced to Float64. Output is Float64. --- datafusion/functions-nested/src/array_sum.rs | 184 ++++++++++++++++++ datafusion/functions-nested/src/lib.rs | 3 + .../sqllogictest/test_files/array_sum.slt | 152 +++++++++++++++ 3 files changed, 339 insertions(+) create mode 100644 datafusion/functions-nested/src/array_sum.rs create mode 100644 datafusion/sqllogictest/test_files/array_sum.slt diff --git a/datafusion/functions-nested/src/array_sum.rs b/datafusion/functions-nested/src/array_sum.rs new file mode 100644 index 0000000000000..6314a64813fcf --- /dev/null +++ b/datafusion/functions-nested/src/array_sum.rs @@ -0,0 +1,184 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! [`ScalarUDFImpl`] definitions for array_sum function. + +use crate::utils::make_scalar_function; +use arrow::array::{Array, ArrayRef, Float64Array, OffsetSizeTrait}; +use arrow::datatypes::{ + DataType, + DataType::{FixedSizeList, LargeList, List, Null}, + Field, +}; +use datafusion_common::cast::{as_float64_array, as_generic_list_array}; +use datafusion_common::utils::{ListCoercion, coerced_type_with_base_type_only}; +use datafusion_common::{Result, internal_err, plan_err, utils::take_function_args}; +use datafusion_expr::{ + ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, + Volatility, +}; +use datafusion_macros::user_doc; +use std::sync::Arc; + +make_udf_expr_and_func!( + ArraySum, + array_sum, + array, + "returns the sum of elements in a numeric array.", + array_sum_udf +); + +#[user_doc( + doc_section(label = "Array Functions"), + description = "Returns the sum of the elements of the input array, computed as `array[0] + array[1] + ...`. NULL elements are skipped (per SQL aggregate convention). Returns NULL if the input row is NULL or every element is NULL. Returns 0.0 for an empty array.", + syntax_example = "array_sum(array)", + sql_example = r#"```sql +> select array_sum([1.0, 2.0, 3.0]); ++----------------------------+ +| array_sum(List([1.0,2.0,3.0])) | ++----------------------------+ +| 6.0 | ++----------------------------+ +```"#, + argument( + name = "array", + description = "Array expression. Can be a constant, column, or function, and any combination of array operators." + ) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct ArraySum { + signature: Signature, + aliases: Vec, +} + +impl Default for ArraySum { + fn default() -> Self { + Self::new() + } +} + +impl ArraySum { + pub fn new() -> Self { + Self { + signature: Signature::user_defined(Volatility::Immutable), + aliases: vec!["list_sum".to_string()], + } + } +} + +impl ScalarUDFImpl for ArraySum { + fn name(&self) -> &str { + "array_sum" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Float64) + } + + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + let [arg_type] = take_function_args(self.name(), arg_types)?; + let coercion = Some(&ListCoercion::FixedSizedListToList); + + if !matches!(arg_type, Null | List(_) | LargeList(_) | FixedSizeList(..)) { + return plan_err!("{} does not support type {arg_type}", self.name()); + } + + let coerced = if matches!(arg_type, Null) { + List(Arc::new(Field::new_list_field(DataType::Float64, true))) + } else { + coerced_type_with_base_type_only(arg_type, &DataType::Float64, coercion) + }; + + Ok(vec![coerced]) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + make_scalar_function(array_sum_inner)(&args.args) + } + + fn aliases(&self) -> &[String] { + &self.aliases + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } +} + +fn array_sum_inner(args: &[ArrayRef]) -> Result { + let [array] = take_function_args("array_sum", args)?; + match array.data_type() { + List(_) => general_array_sum::(array), + LargeList(_) => general_array_sum::(array), + arg_type => { + internal_err!("array_sum received unexpected type after coercion: {arg_type}") + } + } +} + +fn general_array_sum(array: &ArrayRef) -> Result { + let list_array = as_generic_list_array::(array)?; + let values = as_float64_array(list_array.values())?; + let offsets = list_array.value_offsets(); + + let mut builder = Float64Array::builder(list_array.len()); + + for row in 0..list_array.len() { + if list_array.is_null(row) { + builder.append_null(); + continue; + } + + let start = offsets[row].as_usize(); + let end = offsets[row + 1].as_usize(); + let len = end - start; + + // Empty array: sum is the additive identity. Matches SQL SUM() = 0 + // and DuckDB's list_sum(([]) = 0 conventions. + if len == 0 { + builder.append_value(0.0); + continue; + } + + // `slice` resets the logical offset to 0, so `i` below is 0-based within the slice. + let slice = values.slice(start, len); + + // Skip NULL elements per SQL aggregate convention (matches PostgreSQL + // array_sum, DuckDB list_sum, Spark aggregate). A row with every + // element NULL yields NULL — same behavior as SQL SUM over all-NULL. + let mut sum = 0.0_f64; + let mut any_valid = false; + for i in 0..len { + if !slice.is_null(i) { + sum += slice.value(i); + any_valid = true; + } + } + + if any_valid { + builder.append_value(sum); + } else { + builder.append_null(); + } + } + + Ok(Arc::new(builder.finish())) +} diff --git a/datafusion/functions-nested/src/lib.rs b/datafusion/functions-nested/src/lib.rs index 359aa6c8de39c..5b27e2780481b 100644 --- a/datafusion/functions-nested/src/lib.rs +++ b/datafusion/functions-nested/src/lib.rs @@ -49,6 +49,7 @@ pub mod array_normalize; pub mod array_product; pub mod array_scale; pub mod array_subtract; +pub mod array_sum; pub mod array_transform; pub mod arrays_zip; pub mod cardinality; @@ -103,6 +104,7 @@ pub mod expr_fn { pub use super::array_product::array_product; pub use super::array_scale::array_scale; pub use super::array_subtract::array_subtract; + pub use super::array_sum::array_sum; pub use super::array_transform::array_transform; pub use super::arrays_zip::arrays_zip; pub use super::cardinality::cardinality; @@ -182,6 +184,7 @@ pub fn all_default_nested_functions() -> Vec> { array_product::array_product_udf(), array_scale::array_scale_udf(), array_subtract::array_subtract_udf(), + array_sum::array_sum_udf(), cosine_distance::cosine_distance_udf(), inner_product::inner_product_udf(), distance::array_distance_udf(), diff --git a/datafusion/sqllogictest/test_files/array_sum.slt b/datafusion/sqllogictest/test_files/array_sum.slt new file mode 100644 index 0000000000000..740ecce76e054 --- /dev/null +++ b/datafusion/sqllogictest/test_files/array_sum.slt @@ -0,0 +1,152 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +## array_sum + +# Basic case +query R +select array_sum([1.0, 2.0, 3.0]); +---- +6 + +# Single element +query R +select array_sum([5.0]); +---- +5 + +# Negative values +query R +select array_sum([-1.0, -2.0, -3.0]); +---- +-6 + +# Positive and negative cancel +query R +select array_sum([1.0, -1.0, 2.0, -2.0]); +---- +0 + +# Empty array returns 0 (additive identity, per SQL SUM convention) +query R +select array_sum(arrow_cast(make_array(), 'List(Float64)')); +---- +0 + +# Bare NULL input returns NULL row +query R +select array_sum(NULL); +---- +NULL + +# NULL elements are skipped (SQL aggregate convention) +query R +select array_sum([1.0, NULL, 3.0]); +---- +4 + +# Single NULL among numeric: skip the NULL +query R +select array_sum([NULL, 10.0]); +---- +10 + +# All-NULL array returns NULL row (matches SQL SUM over all-NULL) +query R +select array_sum(arrow_cast([NULL, NULL], 'List(Float64)')); +---- +NULL + +# LargeList support +query R +select array_sum(arrow_cast([1.0, 2.0, 3.0], 'LargeList(Float64)')); +---- +6 + +# FixedSizeList input (coerced to List) +query R +select array_sum(arrow_cast([1.0, 2.0, 3.0], 'FixedSizeList(3, Float64)')); +---- +6 + +# Float32 inner type (coerced to Float64) +query R +select array_sum(arrow_cast([1.0, 2.0, 3.0], 'List(Float32)')); +---- +6 + +# Int64 inner type (coerced to Float64) +query R +select array_sum(arrow_cast([1, 2, 3], 'List(Int64)')); +---- +6 + +# Integer literals (coerced to Float64) +query R +select array_sum([1, 2, 3]); +---- +6 + +# Unsupported non-list input (plan error) +query error array_sum does not support type +select array_sum(1); + +# Multi-row query with mix of normal, single-element, NULL elements, empty, NULL row +query R +select array_sum(column1) from (values + (make_array(1.0, 2.0, 3.0)), + (make_array(0.0)), + (make_array(1.0, NULL, 4.0)), + (arrow_cast(make_array(), 'List(Float64)')), + (NULL) +) as t(column1); +---- +6 +0 +5 +0 +NULL + +# Wrong arity (zero args) +query error array_sum function requires 1 argument, got 0 +select array_sum(); + +# Wrong arity (two args) +query error array_sum function requires 1 argument, got 2 +select array_sum([1.0], [2.0]); + +# Return type is Float64 +query RT +select array_sum([1.0, 2.0, 3.0]), arrow_typeof(array_sum([1.0, 2.0, 3.0])); +---- +6 Float64 + +# list_sum alias produces the same result +query R +select list_sum([1.0, 2.0, 3.0]); +---- +6 + +# list_sum alias with NULL row propagates correctly +query R +select list_sum(column1) from (values + (make_array(1.0, 2.0)), + (NULL) +) as t(column1); +---- +3 +NULL From 0d771b163f07553cfd615775c7c08db315b49232 Mon Sep 17 00:00:00 2001 From: Christian McArthur Date: Sun, 7 Jun 2026 20:05:55 -0400 Subject: [PATCH 2/4] refactor: drop intermediate slice + match array_product empty-array semantics Address Jefffrey's review on #22542: - Drop the per-row Float64Array::slice allocation; iterate start..end directly on values - Use values.is_valid(i) instead of !slice.is_null(i) Also pre-empt the empty-array question Jefffrey raised on the merged sibling #22703 (array_product): empty arrays now return NULL via the existing any_valid=false path, matching PostgreSQL, DuckDB list_sum, and SQL Standard SUM-of-empty-set. Updated SLT empty-array expectations and PR body accordingly. Co-Authored-By: Claude Opus 4.7 --- datafusion/functions-nested/src/array_sum.rs | 22 +++++-------------- .../sqllogictest/test_files/array_sum.slt | 6 ++--- 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/datafusion/functions-nested/src/array_sum.rs b/datafusion/functions-nested/src/array_sum.rs index 6314a64813fcf..da2d4f9b0069d 100644 --- a/datafusion/functions-nested/src/array_sum.rs +++ b/datafusion/functions-nested/src/array_sum.rs @@ -149,26 +149,16 @@ fn general_array_sum(array: &ArrayRef) -> Result { let start = offsets[row].as_usize(); let end = offsets[row + 1].as_usize(); - let len = end - start; - - // Empty array: sum is the additive identity. Matches SQL SUM() = 0 - // and DuckDB's list_sum(([]) = 0 conventions. - if len == 0 { - builder.append_value(0.0); - continue; - } - - // `slice` resets the logical offset to 0, so `i` below is 0-based within the slice. - let slice = values.slice(start, len); // Skip NULL elements per SQL aggregate convention (matches PostgreSQL - // array_sum, DuckDB list_sum, Spark aggregate). A row with every - // element NULL yields NULL — same behavior as SQL SUM over all-NULL. + // array_sum, DuckDB list_sum, Spark aggregate). Empty arrays and + // all-NULL arrays both yield NULL — same behavior as SQL SUM over + // an empty set or all-NULL column. let mut sum = 0.0_f64; let mut any_valid = false; - for i in 0..len { - if !slice.is_null(i) { - sum += slice.value(i); + for i in start..end { + if values.is_valid(i) { + sum += values.value(i); any_valid = true; } } diff --git a/datafusion/sqllogictest/test_files/array_sum.slt b/datafusion/sqllogictest/test_files/array_sum.slt index 740ecce76e054..823d767c48489 100644 --- a/datafusion/sqllogictest/test_files/array_sum.slt +++ b/datafusion/sqllogictest/test_files/array_sum.slt @@ -41,11 +41,11 @@ select array_sum([1.0, -1.0, 2.0, -2.0]); ---- 0 -# Empty array returns 0 (additive identity, per SQL SUM convention) +# Empty array returns NULL (matches PostgreSQL, DuckDB list_sum, SQL Standard SUM-of-empty-set) query R select array_sum(arrow_cast(make_array(), 'List(Float64)')); ---- -0 +NULL # Bare NULL input returns NULL row query R @@ -118,7 +118,7 @@ select array_sum(column1) from (values 6 0 5 -0 +NULL NULL # Wrong arity (zero args) From a20165fe08ed882da04c2678e881d7256743ab2c Mon Sep 17 00:00:00 2001 From: Christian McArthur Date: Sun, 7 Jun 2026 20:19:05 -0400 Subject: [PATCH 3/4] docs: regen scalar_functions.md with array_sum entry After rebasing onto apache/main, the scalar_functions.md merge conflict was resolved by taking main's version. Re-ran dev/update_function_docs.sh to reinsert the array_sum entry (with NULL empty-array semantics). Co-Authored-By: Claude Opus 4.7 --- .../source/user-guide/sql/scalar_functions.md | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md index d7026eec09898..73b0e8c6fd39d 100644 --- a/docs/source/user-guide/sql/scalar_functions.md +++ b/docs/source/user-guide/sql/scalar_functions.md @@ -3292,6 +3292,7 @@ _Alias of [current_date](#current_date)._ - [array_slice](#array_slice) - [array_sort](#array_sort) - [array_subtract](#array_subtract) +- [array_sum](#array_sum) - [array_to_string](#array_to_string) - [array_transform](#array_transform) - [array_union](#array_union) @@ -3351,6 +3352,7 @@ _Alias of [current_date](#current_date)._ - [list_slice](#list_slice) - [list_sort](#list_sort) - [list_subtract](#list_subtract) +- [list_sum](#list_sum) - [list_to_string](#list_to_string) - [list_transform](#list_transform) - [list_union](#list_union) @@ -4572,6 +4574,33 @@ array_subtract(array1, array2) - list_subtract +### `array_sum` + +Returns the sum of the elements of the input array, computed as `array[0] + array[1] + ...`. NULL elements are skipped (per SQL aggregate convention). Returns NULL if the input row is NULL or every element is NULL. Returns 0.0 for an empty array. + +```sql +array_sum(array) +``` + +#### Arguments + +- **array**: Array expression. Can be a constant, column, or function, and any combination of array operators. + +#### Example + +```sql +> select array_sum([1.0, 2.0, 3.0]); ++----------------------------+ +| array_sum(List([1.0,2.0,3.0])) | ++----------------------------+ +| 6.0 | ++----------------------------+ +``` + +#### Aliases + +- list_sum + ### `array_to_string` Converts each element to its text representation. @@ -5052,6 +5081,10 @@ _Alias of [array_sort](#array_sort)._ _Alias of [array_subtract](#array_subtract)._ +### `list_sum` + +_Alias of [array_sum](#array_sum)._ + ### `list_to_string` _Alias of [array_to_string](#array_to_string)._ From 512992e430af61734ee8b30503c897389f79b70f Mon Sep 17 00:00:00 2001 From: Christian McArthur Date: Mon, 8 Jun 2026 08:18:09 -0400 Subject: [PATCH 4/4] =?UTF-8?q?fix:=20array=5Fsum=20docstring=20=E2=80=94?= =?UTF-8?q?=20empty=20array=20returns=20NULL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The round-2 refactor changed the empty-array behavior from 0.0 to NULL in the Rust code and SLT, but missed the #[user_doc] description macro which still claimed "Returns 0.0 for an empty array." That stale text propagated through dev/update_function_docs.sh into scalar_functions.md. Updated the docstring to match the actual behavior; re-ran the doc regen to refresh scalar_functions.md. Co-Authored-By: Claude Opus 4.7 --- datafusion/functions-nested/src/array_sum.rs | 2 +- docs/source/user-guide/sql/scalar_functions.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion/functions-nested/src/array_sum.rs b/datafusion/functions-nested/src/array_sum.rs index da2d4f9b0069d..d115355f5cbb9 100644 --- a/datafusion/functions-nested/src/array_sum.rs +++ b/datafusion/functions-nested/src/array_sum.rs @@ -44,7 +44,7 @@ make_udf_expr_and_func!( #[user_doc( doc_section(label = "Array Functions"), - description = "Returns the sum of the elements of the input array, computed as `array[0] + array[1] + ...`. NULL elements are skipped (per SQL aggregate convention). Returns NULL if the input row is NULL or every element is NULL. Returns 0.0 for an empty array.", + description = "Returns the sum of the elements of the input array, computed as `array[0] + array[1] + ...`. NULL elements are skipped (per SQL aggregate convention). Returns NULL if the input row is NULL, every element is NULL, or the array is empty.", syntax_example = "array_sum(array)", sql_example = r#"```sql > select array_sum([1.0, 2.0, 3.0]); diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md index 73b0e8c6fd39d..e5cd6f3d99711 100644 --- a/docs/source/user-guide/sql/scalar_functions.md +++ b/docs/source/user-guide/sql/scalar_functions.md @@ -4576,7 +4576,7 @@ array_subtract(array1, array2) ### `array_sum` -Returns the sum of the elements of the input array, computed as `array[0] + array[1] + ...`. NULL elements are skipped (per SQL aggregate convention). Returns NULL if the input row is NULL or every element is NULL. Returns 0.0 for an empty array. +Returns the sum of the elements of the input array, computed as `array[0] + array[1] + ...`. NULL elements are skipped (per SQL aggregate convention). Returns NULL if the input row is NULL, every element is NULL, or the array is empty. ```sql array_sum(array)