From 2c64548ce53da53fe797ded1e6636b9a9963da4b Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Sat, 3 Oct 2026 13:56:48 +0300 Subject: [PATCH 1/7] fix(arrow): raise `InvalidOperationError` for mismatched `list.contains` items PyArrow, pandas and Dask leaked pyarrow's `ArrowNotImplementedError` when the item cannot be compared with the inner dtype (e.g. a bool or str item on an int list). Raise `InvalidOperationError` instead, as Polars and `ArrowSeries.__contains__` do. --- src/narwhals/_arrow/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/narwhals/_arrow/utils.py b/src/narwhals/_arrow/utils.py index be8856c8b5..2014943e6d 100644 --- a/src/narwhals/_arrow/utils.py +++ b/src/narwhals/_arrow/utils.py @@ -9,7 +9,7 @@ from narwhals._compliant import EagerSeriesNamespace from narwhals._utils import Implementation, Version, isinstance_or_issubclass -from narwhals.exceptions import ColumnNotFoundError +from narwhals.exceptions import ColumnNotFoundError, InvalidOperationError if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Mapping, Sequence @@ -620,7 +620,11 @@ def _list_block_contains(block: ListArrayAny, item: NonNestedLiteral) -> pa.Bool elif isinstance(item, float) and math.isnan(item): matches = pc.is_nan(values) # NaN matches NaN, as in Polars. else: - matches = pc.equal(values, lit(item)) + try: + matches = pc.equal(values, lit(item)) + except pa.ArrowNotImplementedError as exc: + msg = f"Unable to compare item of type {type(item)} with list of type {block.type}." + raise InvalidOperationError(msg) from exc # Counting modulo 2^k stays exact within lists shorter than 2^k, so the narrowest # type that fits the longest list is enough. From 9e34ef1908091dd3e49c91e861b17f644c487153 Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Sat, 3 Oct 2026 13:56:48 +0300 Subject: [PATCH 2/7] test(list-contains): cover numeric coercion, mismatched items and all-null inner Pin how each backend handles mixed numeric kinds (unspecified, see #3900), check that mismatched bool/str/datetime-precision items raise where Polars does, and that an all-null inner list contains nothing. SQL backends coerce mismatched items instead, and pyarrow-backed ones coerce the datetime precision, hence the xfails. --- tests/expr_and_series/list/contains_test.py | 128 +++++++++++++++++++- 1 file changed, 126 insertions(+), 2 deletions(-) diff --git a/tests/expr_and_series/list/contains_test.py b/tests/expr_and_series/list/contains_test.py index 191714f275..be0f9c84b2 100644 --- a/tests/expr_and_series/list/contains_test.py +++ b/tests/expr_and_series/list/contains_test.py @@ -1,22 +1,38 @@ from __future__ import annotations import math -from datetime import date +from datetime import date, datetime from decimal import Decimal from typing import TYPE_CHECKING, Any import pytest import narwhals as nw -from tests.utils import PANDAS_VERSION, POLARS_VERSION, assert_equal_data, pyspark_session +from narwhals.exceptions import InvalidOperationError +from tests.utils import ( + PANDAS_VERSION, + POLARS_VERSION, + assert_equal_data, + maybe_collect, + pyspark_session, +) if TYPE_CHECKING: from narwhals.dtypes import DType + from narwhals.typing import IntoDType, NonNestedLiteral from tests.utils import Constructor, ConstructorEager data = {"a": [[2, 2, 3, None, None], None, []]} expected = {"a": [True, None, False]} +INT_LIST_DATA = {"a": [[2, 2, 3, None, None], None, [], [None], [1], [0, -1]]} + +SQL_BACKENDS = ("duckdb", "ibis", "pyspark", "sqlframe") +"""Backends which coerce the item to the inner dtype instead of matching polars.""" + +ARROW_BACKED_BACKENDS = ("dask", "modin", "pandas", "pyarrow") +"""Backends which compare via pyarrow, which casts a datetime item to the list's unit.""" + def skip_or_xfail_unsupported( request: pytest.FixtureRequest, constructor: Constructor | ConstructorEager @@ -37,6 +53,114 @@ def test_contains_expr(request: pytest.FixtureRequest, constructor: Constructor) assert_equal_data(result, expected) +@pytest.mark.parametrize( + ("data", "inner", "item", "expected"), + [ + pytest.param( + INT_LIST_DATA, + nw.Int64(), + 2.0, + [True, None, False, False, False, False], + id="float_matches_int", + ), + pytest.param( + INT_LIST_DATA, + nw.Int64(), + 1.5, + [False, None, False, False, False, False], + id="non_integer", + ), + pytest.param( + INT_LIST_DATA, + nw.Int8(), + 300, + [False, None, False, False, False, False], + id="overflow", + ), + pytest.param( + {"a": [[1.0, 2.0], [3.0], None, []]}, + nw.Float64(), + 1, + [True, False, None, False], + id="int_matches_float", + ), + ], +) +def test_contains_numeric_coercion_expr( + request: pytest.FixtureRequest, + constructor: Constructor, + data: dict[str, Any], + inner: IntoDType, + item: float, + expected: list[bool | None], +) -> None: + # Mixing numeric kinds is deliberately unspecified, see + # https://github.com/narwhals-dev/narwhals/issues/3900. This pins what each backend + # does today so a future change is visible, it is not a guarantee. + # Polars 2.0 requires an explicit cast rather than lossily coercing to `Float64`; + # `overflow` compares two integer types, so it still resolves. + if ( + "polars" in str(constructor) + and POLARS_VERSION >= (2,) + and "overflow" not in request.node.callspec.id + ): + request.applymarker(pytest.mark.xfail(reason="polars 2.0 needs a cast")) + skip_or_xfail_unsupported(request, constructor) + result = nw.from_native(constructor(data)).select( + nw.col("a").cast(nw.List(inner)).list.contains(item) + ) + assert_equal_data(result, {"a": expected}) + + +@pytest.mark.parametrize( + ("data", "inner", "item"), + [ + pytest.param({"a": [[2, 3], [1], None]}, nw.Int64(), True, id="bool_item"), + pytest.param({"a": [[2, 3], [1], None]}, nw.Int64(), "2", id="str_item"), + pytest.param( + {"a": [[datetime(2020, 1, 1, 1, 2, 3)], [], None]}, + nw.Datetime("ns"), + datetime(2020, 1, 1, 1, 2, 3), + id="datetime_precision", + ), + ], +) +def test_contains_invalid_item_raises( + request: pytest.FixtureRequest, + constructor: Constructor, + data: dict[str, Any], + inner: IntoDType, + item: NonNestedLiteral, +) -> None: + if ( + "datetime_precision" in request.node.callspec.id + and "polars" in str(constructor) + and POLARS_VERSION < (1, 28, 0) + ): + request.applymarker(pytest.mark.xfail(reason="old polars coerces precision")) + coercing_backends = SQL_BACKENDS + if "datetime_precision" in request.node.callspec.id: + coercing_backends += ARROW_BACKED_BACKENDS + if any(backend in str(constructor) for backend in coercing_backends): + request.applymarker(pytest.mark.xfail(reason="mismatched item coerced")) + skip_or_xfail_unsupported(request, constructor) + df = nw.from_native(constructor(data)) + with pytest.raises(InvalidOperationError): + maybe_collect(df.select(nw.col("a").cast(nw.List(inner)).list.contains(item))) + + +def test_contains_all_null_inner_expr( + request: pytest.FixtureRequest, constructor: Constructor +) -> None: + if any(backend in str(constructor) for backend in ("ibis", "pyspark")): + pytest.skip(reason="cannot infer the type of an all-null column") + skip_or_xfail_unsupported(request, constructor) + result = nw.from_native(constructor({"a": [[None, None]]})).select( + nw.col("a").cast(nw.List(nw.Int64())).list.contains(1) + ) + assert_equal_data(result, {"a": [False]}) + + def test_contains_no_match_with_null_elements_expr( request: pytest.FixtureRequest, constructor: Constructor ) -> None: From 9020e18fd721627e2f81a3729b8a0def955fcf86 Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Sat, 3 Oct 2026 18:10:16 +0300 Subject: [PATCH 3/7] fix(arrow): check the `list.contains` item type before processing any data Probe the item against a single null of the inner type, so that a NaN item on a non-float list, a timezone mismatch and empty input all raise `InvalidOperationError` instead of leaking pyarrow errors or returning silently. An empty probe isn't enough, as pyarrow skips its timezone check on empty arrays. Tests: add a NaN item and a zero-row case, select xfails by item value instead of param id, and xfail the NaN case on polars<1.28, which doesn't raise there. Co-authored-by: Francesco Bruzzesi <42817048+FBruzzesi@users.noreply.github.com> --- src/narwhals/_arrow/utils.py | 29 +++++++++----- tests/expr_and_series/list/contains_test.py | 44 +++++++++++---------- 2 files changed, 43 insertions(+), 30 deletions(-) diff --git a/src/narwhals/_arrow/utils.py b/src/narwhals/_arrow/utils.py index 2014943e6d..463c88a233 100644 --- a/src/narwhals/_arrow/utils.py +++ b/src/narwhals/_arrow/utils.py @@ -575,10 +575,28 @@ def list_contains(array: ChunkedArrayAny, item: NonNestedLiteral) -> ChunkedArra A running count of matches over the flattened values changes within a list iff the list holds a match. That keeps this linear, where a group-by per list would sort. """ + list_type = cast("pa.ListType[Any] | pa.LargeListType[Any]", array.type) + try: + # Probe a single null, so that a mismatched `item` raises even without rows. + # Not an empty array: pyarrow skips its timezone check on those. + _item_matches(pa.nulls(1, list_type.value_type), item) + except (pa.ArrowInvalid, pa.ArrowNotImplementedError, pa.ArrowTypeError) as exc: + msg = ( + f"Unable to compare item of type {type(item)} with list of type {list_type}." + ) + raise InvalidOperationError(msg) from exc blocks = [_list_block_contains(block, item) for block in _list_blocks(array)] return pa.chunked_array(blocks, pa.bool_()) +def _item_matches(values: ArrayAny, item: NonNestedLiteral) -> pa.BooleanArray: + if item is None: + return pc.is_null(values) + if isinstance(item, float) and math.isnan(item): + return pc.is_nan(values) # NaN matches NaN, as in Polars. + return pc.equal(values, lit(item)) + + def _list_blocks(array: ChunkedArrayAny) -> Iterator[ListArrayAny]: """Split `array` into runs of whole lists, of up to about `_LIST_BLOCK_VALUES` values. @@ -615,16 +633,7 @@ def _list_block_contains(block: ListArrayAny, item: NonNestedLiteral) -> pa.Bool offsets = pc.subtract(offsets, lit(first, offsets.type)) # type: ignore[arg-type] ends, starts = offsets.slice(1), offsets.slice(0, len(block)) - if item is None: - matches = pc.is_null(values) - elif isinstance(item, float) and math.isnan(item): - matches = pc.is_nan(values) # NaN matches NaN, as in Polars. - else: - try: - matches = pc.equal(values, lit(item)) - except pa.ArrowNotImplementedError as exc: - msg = f"Unable to compare item of type {type(item)} with list of type {block.type}." - raise InvalidOperationError(msg) from exc + matches = _item_matches(values, item) # Counting modulo 2^k stays exact within lists shorter than 2^k, so the narrowest # type that fits the longest list is enough. diff --git a/tests/expr_and_series/list/contains_test.py b/tests/expr_and_series/list/contains_test.py index be0f9c84b2..2bee9af915 100644 --- a/tests/expr_and_series/list/contains_test.py +++ b/tests/expr_and_series/list/contains_test.py @@ -19,7 +19,7 @@ if TYPE_CHECKING: from narwhals.dtypes import DType - from narwhals.typing import IntoDType, NonNestedLiteral + from narwhals.typing import NonNestedLiteral from tests.utils import Constructor, ConstructorEager data = {"a": [[2, 2, 3, None, None], None, []]} @@ -30,8 +30,8 @@ SQL_BACKENDS = ("duckdb", "ibis", "pyspark", "sqlframe") """Backends which coerce the item to the inner dtype instead of matching polars.""" -ARROW_BACKED_BACKENDS = ("dask", "modin", "pandas", "pyarrow") -"""Backends which compare via pyarrow, which casts a datetime item to the list's unit.""" +PYARROW_COMPUTE_BACKENDS = ("dask", "modin", "pandas", "pyarrow") +"""Backends whose `list.contains` runs on pyarrow, which casts a datetime item to the list's unit.""" def skip_or_xfail_unsupported( @@ -90,20 +90,16 @@ def test_contains_numeric_coercion_expr( request: pytest.FixtureRequest, constructor: Constructor, data: dict[str, Any], - inner: IntoDType, + inner: DType, item: float, expected: list[bool | None], ) -> None: # Mixing numeric kinds is deliberately unspecified, see # https://github.com/narwhals-dev/narwhals/issues/3900. This pins what each backend # does today so a future change is visible, it is not a guarantee. - # Polars 2.0 requires an explicit cast rather than lossily coercing to `Float64`; - # `overflow` compares two integer types, so it still resolves. - if ( - "polars" in str(constructor) - and POLARS_VERSION >= (2,) - and "overflow" not in request.node.callspec.id - ): + # Polars 2.0 requires an explicit cast rather than lossily coercing to `Float64`. + mixes_kinds = isinstance(item, float) != inner.is_float() + if "polars" in str(constructor) and POLARS_VERSION >= (2,) and mixes_kinds: request.applymarker(pytest.mark.xfail(reason="polars 2.0 needs a cast")) skip_or_xfail_unsupported(request, constructor) result = nw.from_native(constructor(data)).select( @@ -117,6 +113,7 @@ def test_contains_numeric_coercion_expr( [ pytest.param({"a": [[2, 3], [1], None]}, nw.Int64(), True, id="bool_item"), pytest.param({"a": [[2, 3], [1], None]}, nw.Int64(), "2", id="str_item"), + pytest.param({"a": [["x"], [], None]}, nw.String(), float("nan"), id="nan_item"), pytest.param( {"a": [[datetime(2020, 1, 1, 1, 2, 3)], [], None]}, nw.Datetime("ns"), @@ -129,18 +126,14 @@ def test_contains_invalid_item_raises( request: pytest.FixtureRequest, constructor: Constructor, data: dict[str, Any], - inner: IntoDType, + inner: DType, item: NonNestedLiteral, ) -> None: - if ( - "datetime_precision" in request.node.callspec.id - and "polars" in str(constructor) - and POLARS_VERSION < (1, 28, 0) - ): - request.applymarker(pytest.mark.xfail(reason="old polars coerces precision")) coercing_backends = SQL_BACKENDS - if "datetime_precision" in request.node.callspec.id: - coercing_backends += ARROW_BACKED_BACKENDS + if isinstance(item, (float, datetime)) and POLARS_VERSION < (1, 28, 0): + coercing_backends += ("polars",) + if isinstance(item, datetime): + coercing_backends += PYARROW_COMPUTE_BACKENDS if any(backend in str(constructor) for backend in coercing_backends): request.applymarker(pytest.mark.xfail(reason="mismatched item coerced")) skip_or_xfail_unsupported(request, constructor) @@ -149,6 +142,17 @@ def test_contains_invalid_item_raises( maybe_collect(df.select(nw.col("a").cast(nw.List(inner)).list.contains(item))) +def test_contains_invalid_item_raises_without_rows( + request: pytest.FixtureRequest, constructor: Constructor +) -> None: + if any(backend in str(constructor) for backend in SQL_BACKENDS): + request.applymarker(pytest.mark.xfail(reason="mismatched item coerced")) + skip_or_xfail_unsupported(request, constructor) + df = nw.from_native(constructor({"a": [[2, 3]]})).filter(nw.col("a").is_null()) + with pytest.raises(InvalidOperationError): + maybe_collect(df.select(nw.col("a").cast(nw.List(nw.Int64())).list.contains("2"))) + + def test_contains_all_null_inner_expr( request: pytest.FixtureRequest, constructor: Constructor ) -> None: From 4e0c89378852009a1d1932bd5c8326192642c1fe Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Sat, 3 Oct 2026 18:16:33 +0300 Subject: [PATCH 4/7] test(list-contains): mark the old-polars xfail with `applymarker`, so coverage skips it The full-coverage job runs polars>=1.28, so the branch never ran there. --- tests/expr_and_series/list/contains_test.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/expr_and_series/list/contains_test.py b/tests/expr_and_series/list/contains_test.py index 2bee9af915..8fd22236f5 100644 --- a/tests/expr_and_series/list/contains_test.py +++ b/tests/expr_and_series/list/contains_test.py @@ -129,9 +129,13 @@ def test_contains_invalid_item_raises( inner: DType, item: NonNestedLiteral, ) -> None: + if ( + isinstance(item, (float, datetime)) + and "polars" in str(constructor) + and POLARS_VERSION < (1, 28, 0) + ): + request.applymarker(pytest.mark.xfail(reason="old polars coerces the item")) coercing_backends = SQL_BACKENDS - if isinstance(item, (float, datetime)) and POLARS_VERSION < (1, 28, 0): - coercing_backends += ("polars",) if isinstance(item, datetime): coercing_backends += PYARROW_COMPUTE_BACKENDS if any(backend in str(constructor) for backend in coercing_backends): From f555010911feb141c8d0e5516cc2370d5c0ff0b2 Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Sat, 10 Oct 2026 22:52:03 +0200 Subject: [PATCH 5/7] test(list-contains): follow Polars 2.0, which coerces datetime units but rejects time-zone mismatches --- tests/expr_and_series/list/contains_test.py | 41 +++++++++++++-------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/tests/expr_and_series/list/contains_test.py b/tests/expr_and_series/list/contains_test.py index ead894fac1..1135cd5ba4 100644 --- a/tests/expr_and_series/list/contains_test.py +++ b/tests/expr_and_series/list/contains_test.py @@ -1,7 +1,7 @@ from __future__ import annotations import math -from datetime import date, datetime +from datetime import date, datetime, timezone from decimal import Decimal from typing import TYPE_CHECKING, Any @@ -30,9 +30,6 @@ SQL_BACKENDS = ("duckdb", "ibis", "pyspark", "sqlframe") """Backends which coerce the item to the inner dtype instead of matching polars.""" -PYARROW_COMPUTE_BACKENDS = ("dask", "modin", "pandas", "pyarrow") -"""Backends whose `list.contains` runs on pyarrow, which casts a datetime item to the list's unit.""" - def skip_or_xfail_unsupported( request: pytest.FixtureRequest, constructor: Constructor | ConstructorEager @@ -116,9 +113,9 @@ def test_contains_numeric_coercion_expr( pytest.param({"a": [["x"], [], None]}, nw.String(), float("nan"), id="nan_item"), pytest.param( {"a": [[datetime(2020, 1, 1, 1, 2, 3)], [], None]}, - nw.Datetime("ns"), - datetime(2020, 1, 1, 1, 2, 3), - id="datetime_precision", + nw.Datetime("us"), + datetime(2020, 1, 1, 1, 2, 3, tzinfo=timezone.utc), + id="datetime_time_zone", ), ], ) @@ -129,16 +126,12 @@ def test_contains_invalid_item_raises( inner: DType, item: NonNestedLiteral, ) -> None: - if ( - isinstance(item, (float, datetime)) - and "polars" in str(constructor) - and POLARS_VERSION < (1, 28, 0) + if "polars" in str(constructor) and ( + (isinstance(item, float) and POLARS_VERSION < (1, 28, 0)) + or (isinstance(item, datetime) and POLARS_VERSION < (2,)) ): request.applymarker(pytest.mark.xfail(reason="old polars coerces the item")) - coercing_backends = SQL_BACKENDS - if isinstance(item, datetime): - coercing_backends += PYARROW_COMPUTE_BACKENDS - if any(backend in str(constructor) for backend in coercing_backends): + if any(backend in str(constructor) for backend in SQL_BACKENDS): request.applymarker(pytest.mark.xfail(reason="mismatched item coerced")) skip_or_xfail_unsupported(request, constructor) df = nw.from_native(constructor(data)) @@ -146,6 +139,24 @@ def test_contains_invalid_item_raises( maybe_collect(df.select(nw.col("a").cast(nw.List(inner)).list.contains(item))) +def test_contains_datetime_precision_coerced( + request: pytest.FixtureRequest, constructor: Constructor +) -> None: + # Polars 1.28 up to 2.0 raises instead, and before 1.28 it ignores the unit. + if "polars" in str(constructor) and POLARS_VERSION < (2,): + request.applymarker(pytest.mark.xfail(reason="old polars mismatches units")) + if any(backend in str(constructor) for backend in ("pyspark", "sqlframe")): + pytest.skip(reason="only microsecond precision is supported") + skip_or_xfail_unsupported(request, constructor) + df = nw.from_native(constructor({"a": [[datetime(2020, 1, 1, 1, 2, 3)], [], None]})) + result = df.select( + nw.col("a") + .cast(nw.List(nw.Datetime("ms"))) + .list.contains(datetime(2020, 1, 1, 1, 2, 3)) + ) + assert_equal_data(result, {"a": [True, False, None]}) + + def test_contains_invalid_item_raises_without_rows( request: pytest.FixtureRequest, constructor: Constructor ) -> None: From e2ddb128fca3e2255bee145d26758258b3814655 Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Sat, 10 Oct 2026 23:11:23 +0200 Subject: [PATCH 6/7] test(list-contains): clarify why polars before 1.28 fails the datetime unit test --- tests/expr_and_series/list/contains_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/expr_and_series/list/contains_test.py b/tests/expr_and_series/list/contains_test.py index 1135cd5ba4..0a4cdf976b 100644 --- a/tests/expr_and_series/list/contains_test.py +++ b/tests/expr_and_series/list/contains_test.py @@ -142,7 +142,8 @@ def test_contains_invalid_item_raises( def test_contains_datetime_precision_coerced( request: pytest.FixtureRequest, constructor: Constructor ) -> None: - # Polars 1.28 up to 2.0 raises instead, and before 1.28 it ignores the unit. + # Polars 1.28 up to 2.0 raises instead, and before 1.28 it compares the raw + # integers without converting the unit, so it never finds a match. if "polars" in str(constructor) and POLARS_VERSION < (2,): request.applymarker(pytest.mark.xfail(reason="old polars mismatches units")) if any(backend in str(constructor) for backend in ("pyspark", "sqlframe")): From 14bec5bcced84fff9776e88d2cc8996d1e503e45 Mon Sep 17 00:00:00 2001 From: FBruzzesi Date: Sat, 10 Oct 2026 23:55:59 +0200 Subject: [PATCH 7/7] describe old-Polars unit behaviour for both list units --- tests/expr_and_series/list/contains_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/expr_and_series/list/contains_test.py b/tests/expr_and_series/list/contains_test.py index 0a4cdf976b..65dd0db1c3 100644 --- a/tests/expr_and_series/list/contains_test.py +++ b/tests/expr_and_series/list/contains_test.py @@ -142,8 +142,8 @@ def test_contains_invalid_item_raises( def test_contains_datetime_precision_coerced( request: pytest.FixtureRequest, constructor: Constructor ) -> None: - # Polars 1.28 up to 2.0 raises instead, and before 1.28 it compares the raw - # integers without converting the unit, so it never finds a match. + # Before 2.0, Polars doesn't convert the item to the list's time unit, so it never + # finds the match (1.28-1.x raises instead when the list's unit is finer). if "polars" in str(constructor) and POLARS_VERSION < (2,): request.applymarker(pytest.mark.xfail(reason="old polars mismatches units")) if any(backend in str(constructor) for backend in ("pyspark", "sqlframe")):