From da67de799b5cab70501d85e4cbb66cb22ef8b652 Mon Sep 17 00:00:00 2001 From: Lingikaushikreddy Date: Mon, 5 Oct 2026 20:36:39 -0700 Subject: [PATCH] Align fractional float casts to signed integers in the main API --- docs/backcompat.md | 5 + src/narwhals/_arrow/series.py | 13 ++- src/narwhals/_arrow/utils.py | 6 ++ src/narwhals/_duckdb/expr.py | 27 ++++- src/narwhals/_ibis/expr.py | 10 ++ src/narwhals/_pandas_like/series.py | 16 ++- src/narwhals/_spark_like/expr.py | 27 +++-- src/narwhals/expr.py | 4 + src/narwhals/series.py | 4 + tests/expr_and_series/cast_test.py | 147 +++++++++++++++++++++++++++- 10 files changed, 241 insertions(+), 18 deletions(-) diff --git a/docs/backcompat.md b/docs/backcompat.md index 81adc89daf..05230bbf2b 100644 --- a/docs/backcompat.md +++ b/docs/backcompat.md @@ -118,6 +118,11 @@ Which should you use? In general we recommend: ## `main` vs `stable.v2` differences +- Casting floating-point values to signed integers in the main namespace truncates + towards zero. `narwhals.stable.v1` and `narwhals.stable.v2` retain backend-native + casting behavior: PyArrow and pandas with Arrow dtypes reject fractional values, + while DuckDB, SQLFrame with DuckDB, and Ibis with DuckDB round them. + - Since Narwhals 2.26: - `Expr.cat.get_categories` and `Series.cat.get_categories` are deprecated in the main diff --git a/src/narwhals/_arrow/series.py b/src/narwhals/_arrow/series.py index 1cb85da0ef..3d6b7ff4fb 100644 --- a/src/narwhals/_arrow/series.py +++ b/src/narwhals/_arrow/series.py @@ -23,6 +23,7 @@ nulls_like, pad_series, sortable, + truncate_float, zeros, ) from narwhals._compliant import EagerSeries, EagerSeriesHist @@ -30,6 +31,7 @@ from narwhals._utils import ( NO_DEFAULT, Implementation, + Version, generate_temporary_column_name, is_list_of, not_implemented, @@ -63,7 +65,7 @@ ) from narwhals._compliant.series import HistData from narwhals._typing import NoDefault - from narwhals._utils import Version, _LimitedContext + from narwhals._utils import _LimitedContext from narwhals.dtypes import DType from narwhals.typing import ( ClosedInterval, @@ -562,7 +564,14 @@ def is_nan(self) -> Self: def cast(self, dtype: IntoDType) -> Self: data_type = narwhals_to_native_dtype(dtype, self._version) - return self._with_native(pc.cast(self.native, data_type), preserve_broadcast=True) + native = self.native + if ( + self._version is Version.MAIN + and self.dtype.is_float() + and dtype.is_signed_integer() + ): + native = truncate_float(native) + return self._with_native(pc.cast(native, data_type), preserve_broadcast=True) def null_count(self, *, _return_py_scalar: bool = True) -> int: return maybe_extract_py_scalar(self.native.null_count, _return_py_scalar) diff --git a/src/narwhals/_arrow/utils.py b/src/narwhals/_arrow/utils.py index be8856c8b5..8d2cc001c0 100644 --- a/src/narwhals/_arrow/utils.py +++ b/src/narwhals/_arrow/utils.py @@ -354,6 +354,12 @@ def floordiv_compat(left: ArrayOrScalar, right: ArrayOrScalar, /) -> Any: return result +def truncate_float(native: ChunkedArrayAny) -> ChunkedArrayAny: + if pa.types.is_float16(native.type): + native = pc.cast(native, pa.float32()) + return pc.trunc(native) + + def cast_for_truediv( arrow_array: ArrayOrScalarT1, pa_object: ArrayOrScalarT2 ) -> tuple[ArrayOrScalarT1, ArrayOrScalarT2]: diff --git a/src/narwhals/_duckdb/expr.py b/src/narwhals/_duckdb/expr.py index 26c6524133..fcdf2bada1 100644 --- a/src/narwhals/_duckdb/expr.py +++ b/src/narwhals/_duckdb/expr.py @@ -279,15 +279,32 @@ def _fill_constant(expr: Expression, value: Any) -> Expression: return self._with_elementwise(_fill_constant, expression_args={"value": value}) def cast(self, dtype: IntoDType) -> Self: - def func(df: DuckDBLazyFrame) -> list[Expression]: + def cast_exprs( + df: DuckDBLazyFrame, exprs: Sequence[Expression], *, aggregated: bool + ) -> list[Expression]: tz = DeferredTimeZone(df.native) native_dtype = narwhals_to_native_dtype(dtype, self._version, tz) - return [expr.cast(native_dtype) for expr in self(df)] + if self._version is Version.MAIN and dtype.is_signed_integer(): + relation = ( + df.native.aggregate(exprs) if aggregated else df.native.select(*exprs) + ) + source_types = relation.types + return [ + ( + F("trunc", expr) + if source_type.id in {"float", "double"} + else expr + ).cast(native_dtype) + for expr, source_type in zip(exprs, source_types, strict=True) + ] + return [expr.cast(native_dtype) for expr in exprs] + + def func(df: DuckDBLazyFrame) -> list[Expression]: + aggregated = self._opt_metadata is not None and self._metadata.is_scalar_like + return cast_exprs(df, self(df), aggregated=aggregated) def window_f(df: DuckDBLazyFrame, inputs: DuckDBWindowInputs) -> list[Expression]: - tz = DeferredTimeZone(df.native) - native_dtype = narwhals_to_native_dtype(dtype, self._version, tz) - return [expr.cast(native_dtype) for expr in self.window_function(df, inputs)] + return cast_exprs(df, self.window_function(df, inputs), aggregated=False) return self.__class__( func, diff --git a/src/narwhals/_ibis/expr.py b/src/narwhals/_ibis/expr.py index 6178b8a618..f66d18ab2c 100644 --- a/src/narwhals/_ibis/expr.py +++ b/src/narwhals/_ibis/expr.py @@ -310,6 +310,16 @@ def _fill_null(expr: ir.Value, value: ir.Scalar) -> ir.Value: def cast(self, dtype: IntoDType) -> Self: def _func(expr: ir.Column) -> ir.Value: native_dtype = narwhals_to_native_dtype(dtype, self._version) + if ( + self._version is Version.MAIN + and dtype.is_signed_integer() + and is_floating(expr.type()) + ): + numeric = cast("ir.FloatingValue", expr) + expr = cast( + "ir.Column", + ibis.ifelse(expr >= lit(0), numeric.floor(), numeric.ceil()), + ) # ibis `cast` overloads do not include DataType, only literals return expr.cast(native_dtype) # type: ignore[unused-ignore] diff --git a/src/narwhals/_pandas_like/series.py b/src/narwhals/_pandas_like/series.py index da402aa909..4ce9c02120 100644 --- a/src/narwhals/_pandas_like/series.py +++ b/src/narwhals/_pandas_like/series.py @@ -29,7 +29,7 @@ set_index, ) from narwhals._typing_compat import assert_never -from narwhals._utils import NO_DEFAULT, Implementation, is_list_of +from narwhals._utils import NO_DEFAULT, Implementation, Version, is_list_of from narwhals.dependencies import is_numpy_array_1d, is_pandas_like_series from narwhals.dtypes import String from narwhals.exceptions import InvalidOperationError @@ -50,7 +50,7 @@ from narwhals._pandas_like.namespace import PandasLikeNamespace from narwhals._pandas_like.typing import NativeSeriesT from narwhals._typing import NoDefault - from narwhals._utils import Version, _LimitedContext + from narwhals._utils import _LimitedContext from narwhals.dtypes import DType from narwhals.typing import ( ClosedInterval, @@ -328,7 +328,17 @@ def cast(self, dtype: IntoDType) -> Self: implementation=self._implementation, version=self._version, ) - result = self.native.astype(pd_dtype) + native = self.native + if ( + self._version is Version.MAIN + and self.dtype.is_float() + and dtype.is_signed_integer() + and self.is_native_dtype_pyarrow(native.dtype) + ): + from narwhals._arrow.utils import truncate_float + + native = self._apply_pyarrow_compute_func(native, truncate_float) + result = native.astype(pd_dtype) if ( pd_dtype is str and is_pandas_or_modin(self._implementation) diff --git a/src/narwhals/_spark_like/expr.py b/src/narwhals/_spark_like/expr.py index 917e1aef23..c5e42da8df 100644 --- a/src/narwhals/_spark_like/expr.py +++ b/src/narwhals/_spark_like/expr.py @@ -258,19 +258,34 @@ def __neg__(self) -> Self: return self._with_elementwise(neg) def cast(self, dtype: IntoDType) -> Self: - def func(df: SparkLikeLazyFrame) -> Sequence[Column]: + def cast_exprs(df: SparkLikeLazyFrame, exprs: Sequence[Column]) -> list[Column]: spark_dtype = narwhals_to_native_dtype( dtype, self._version, self._native_dtypes, df.native.sparkSession ) - return [expr.cast(spark_dtype) for expr in self(df)] + if ( + self._version is Version.MAIN + and self._implementation is Implementation.SQLFRAME + and dtype.is_signed_integer() + ): + fields = df.native.select(*exprs).schema.fields + F = self._F + return [ + ( + F.when(expr >= F.lit(0), F.floor(expr)).otherwise(F.ceil(expr)) + if field.dataType.typeName() in {"float", "double"} + else expr + ).cast(spark_dtype) + for expr, field in zip(exprs, fields, strict=True) + ] + return [expr.cast(spark_dtype) for expr in exprs] + + def func(df: SparkLikeLazyFrame) -> Sequence[Column]: + return cast_exprs(df, self(df)) def window_f( df: SparkLikeLazyFrame, inputs: SparkWindowInputs ) -> Sequence[Column]: - spark_dtype = narwhals_to_native_dtype( - dtype, self._version, self._native_dtypes, df.native.sparkSession - ) - return [expr.cast(spark_dtype) for expr in self.window_function(df, inputs)] + return cast_exprs(df, self.window_function(df, inputs)) return self.__class__( func, diff --git a/src/narwhals/expr.py b/src/narwhals/expr.py index b6d5fb251c..779896fc4e 100644 --- a/src/narwhals/expr.py +++ b/src/narwhals/expr.py @@ -172,6 +172,10 @@ def pipe( def cast(self, dtype: IntoDType) -> Self: """Redefine an object's data type. + In the main `narwhals` namespace, floating-point values are truncated towards + zero when casting to signed integers. Stable namespaces retain backend-native + casting behavior. + Arguments: dtype: Data type that the object will be cast into. diff --git a/src/narwhals/series.py b/src/narwhals/series.py index 30a4715ddc..79d99f506d 100644 --- a/src/narwhals/series.py +++ b/src/narwhals/series.py @@ -617,6 +617,10 @@ def ewm_mean( def cast(self, dtype: IntoDType) -> Self: """Cast between data types. + In the main `narwhals` namespace, floating-point values are truncated towards + zero when casting to signed integers. Stable namespaces retain backend-native + casting behavior. + Arguments: dtype: Data type that the object will be cast into. diff --git a/tests/expr_and_series/cast_test.py b/tests/expr_and_series/cast_test.py index 59132f09a2..343f3c140c 100644 --- a/tests/expr_and_series/cast_test.py +++ b/tests/expr_and_series/cast_test.py @@ -6,7 +6,9 @@ import pytest import narwhals as nw -from narwhals.exceptions import UnsupportedDTypeError +import narwhals.stable.v1 as nw_v1 +import narwhals.stable.v2 as nw_v2 +from narwhals.exceptions import InvalidOperationError, UnsupportedDTypeError from tests.utils import ( PANDAS_VERSION, POLARS_VERSION, @@ -21,7 +23,7 @@ if TYPE_CHECKING: from collections.abc import Mapping - from narwhals.typing import NonNestedDType + from narwhals.typing import IntoDType, NonNestedDType DATA = { "a": [1], @@ -66,6 +68,147 @@ MODIN_XFAIL_COLUMNS = {"o", "k"} +@pytest.mark.parametrize("dtype", [nw.Int8, nw.Int16(), nw.Int32, nw.Int64()]) +@pytest.mark.parametrize("float_dtype", [nw.Float32, nw.Float64]) +def test_cast_float_to_signed_integer( + constructor: Constructor, dtype: IntoDType, float_dtype: type[nw.Float32 | nw.Float64] +) -> None: + values = [1.7, -1.7, 2.5, -2.5, 0.9, -0.9] + data = {"idx": list(range(len(values))), "a": values} + df = nw.from_native(constructor(data)) + result = df.select("idx", nw.col("a").cast(float_dtype).cast(dtype)).sort("idx") + assert_equal_data(result, {"idx": data["idx"], "a": [1, -1, 2, -2, 0, 0]}) + assert result.collect_schema()["a"] == dtype + assert_equal_data(df, data) + + +@pytest.mark.parametrize("dtype", [nw.Int8, nw.Int16, nw.Int32, nw.Int64]) +def test_cast_float_to_signed_integer_series( + constructor_eager: ConstructorEager, dtype: IntoDType +) -> None: + s = nw.from_native(constructor_eager({"a": [1.7, -1.7, 2.5, -2.5, 0.9, -0.9]}))["a"] + result = s.cast(dtype) + assert result.to_list() == [1, -1, 2, -2, 0, 0] + assert result.dtype == dtype + assert s.to_list() == [1.7, -1.7, 2.5, -2.5, 0.9, -0.9] + + +def test_cast_float_to_signed_integer_nulls(constructor: Constructor) -> None: + if any( + x in str(constructor) for x in ("pandas_constructor", "modin_constructor", "dask") + ): + pytest.skip("NumPy-backed integers cannot represent nulls") + df = nw.from_native(constructor({"idx": [0, 1, 2], "a": [1.7, None, -1.7]})) + result = df.select("idx", nw.col("a").cast(nw.Int64)).sort("idx") + assert_equal_data(result, {"idx": [0, 1, 2], "a": [1, None, -1]}) + + +def test_frame_cast_float_to_signed_integer(constructor: Constructor) -> None: + if "duckdb" not in str(constructor): + pytest.skip("DuckDB frame casts build compliant expressions without metadata") + df = nw.from_native(constructor({"idx": [0, 1], "a": [1.7, -1.7]})) + result = df.cast({"a": nw.Int64}).sort("idx") + assert_equal_data(result, {"idx": [0, 1], "a": [1, -1]}) + + +@pytest.mark.parametrize( + "values", [[128.0], [-129.0], [float("inf")], [float("-inf")], [1e30]] +) +def test_cast_float_to_signed_integer_overflow( + constructor: Constructor, values: list[float] +) -> None: + if "pyarrow" in str(constructor): + exception = pytest.importorskip("pyarrow").ArrowInvalid + elif any(x in str(constructor) for x in ("duckdb", "sqlframe")): + exception = pytest.importorskip("duckdb").ConversionException + elif "polars" in str(constructor): + exception = InvalidOperationError + else: + pytest.skip("These backends perform checked integer casts") + df = nw.from_native(constructor({"a": values})) + with pytest.raises(exception, match=r"convert|cast|range|truncate"): + df.select(nw.col("a").cast(nw.Int8)).lazy().collect() + + +@pytest.mark.parametrize("nw_stable", [nw_v1, nw_v2]) +def test_cast_float_to_signed_integer_stable( + constructor: Constructor, nw_stable: Any +) -> None: + if nw_stable is nw_v1 and "duckdb" in str(constructor): + pytest.skip("DuckDB is interchange-only in stable.v1") + df = nw_stable.from_native(constructor({"idx": [0, 1], "a": [1.7, -1.7]})) + if "pyarrow" in str(constructor): + exception = pytest.importorskip("pyarrow").ArrowInvalid + with pytest.raises(exception, match="truncated"): + df.select(nw_stable.col("a").cast(nw_stable.Int64)).lazy().collect() + else: + expected = ( + [2, -2] + if any(x in str(constructor) for x in ("duckdb", "sqlframe", "ibis")) + else [1, -1] + ) + result = df.select("idx", nw_stable.col("a").cast(nw_stable.Int64)).sort("idx") + assert_equal_data(result, {"idx": [0, 1], "a": expected}) + + +def test_cast_float_to_signed_integer_over(constructor: Constructor) -> None: + df = nw.from_native( + constructor({"idx": [0, 1, 2], "g": [1, 1, 2], "a": [1.7, 1.7, -1.7]}) + ) + result = df.select("idx", nw.col("a").mean().cast(nw.Int64).over("g")).sort("idx") + assert_equal_data(result, {"idx": [0, 1, 2], "a": [1, 1, -1]}) + + +@pytest.mark.parametrize("value", [1.7, -1.7]) +@pytest.mark.parametrize("grouped", [False, True]) +def test_cast_float_aggregate_to_signed_integer( + constructor: Constructor, value: float, *, grouped: bool +) -> None: + if not any(x in str(constructor) for x in ("duckdb", "sqlframe", "ibis")): + pytest.skip("Source-type binding for aggregate casts is SQL-backend specific") + df = nw.from_native(constructor({"g": [1, 1, 2, 2], "a": [value] * 4})) + expr = nw.col("a").mean().cast(nw.Int64) + if grouped: + result = df.group_by("g").agg(expr).sort("g") + expected = {"g": [1, 2], "a": [int(value)] * 2} + else: + result = df.select(expr) + expected = {"a": [int(value)]} + assert_equal_data(result, expected) + + +@pytest.mark.parametrize("values", [["1", "2"], [2**53 + 1, 2**53 + 3]]) +@pytest.mark.parametrize("grouped", [False, True]) +def test_cast_nonfloat_aggregate_to_signed_integer( + constructor: Constructor, values: list[str] | list[int], *, grouped: bool +) -> None: + if not any(x in str(constructor) for x in ("duckdb", "sqlframe", "ibis")): + pytest.skip("Source-type binding for aggregate casts is SQL-backend specific") + df = nw.from_native(constructor({"g": [1, 1], "a": values})) + expr = nw.col("a").max().cast(nw.Int64) + if grouped: + result = df.group_by("g").agg(expr) + expected = {"g": [1], "a": [int(values[1])]} + else: + result = df.select(expr) + expected = {"a": [int(values[1])]} + assert_equal_data(result, expected) + + +def test_cast_float16_to_signed_integer(constructor_eager: ConstructorEager) -> None: + if "pyarrow" not in str(constructor_eager) or PYARROW_VERSION < (16,): + pytest.skip("PyArrow added half-float casting in 16.0") + s = nw.from_native(constructor_eager({"a": [1.7, None, -1.7]}))["a"].cast(nw.Float16) + assert_equal_data(s.cast(nw.Int64).to_frame(), {"a": [1, None, -1]}) + + +def test_cast_float_nan_to_signed_integer_pyarrow() -> None: + pa = pytest.importorskip("pyarrow") + s = nw.from_native(pa.chunked_array([[float("nan")]]), series_only=True) + with pytest.raises(pa.ArrowInvalid, match="truncated"): + s.cast(nw.Int64) + + @pytest.mark.filterwarnings("ignore:casting period[M] values to int64:FutureWarning") def test_cast(constructor: Constructor) -> None: if "pyarrow_table_constructor" in str(constructor) and PYARROW_VERSION <= (