diff --git a/doc/reference/param/parameters.md b/doc/reference/param/parameters.md index cb9f48261..dc49d0ebb 100644 --- a/doc/reference/param/parameters.md +++ b/doc/reference/param/parameters.md @@ -65,6 +65,7 @@ Array Series DataFrame + DataFrameLike Callable Action Composite diff --git a/param/__init__.py b/param/__init__.py index 4aa051eef..989f7659f 100644 --- a/param/__init__.py +++ b/param/__init__.py @@ -84,6 +84,7 @@ Dict, Array, DataFrame, + DataFrameLike, Series, Path, Filename, @@ -166,6 +167,7 @@ 'Composite', 'DEBUG', 'DataFrame', + 'DataFrameLike', 'Date', 'DateRange', 'Dict', diff --git a/param/_utils.py b/param/_utils.py index 11870b935..84fbcfc24 100644 --- a/param/_utils.py +++ b/param/_utils.py @@ -699,3 +699,15 @@ def _find_stack_level() -> int: # https://docs.python.org/3/library/inspect.html#inspect.Traceback del frame return n + + +def _get_narwhals(): + """Import and return the optional ``narwhals`` stable API.""" + try: + import narwhals.stable.v2 as narwhals + except ModuleNotFoundError as e: + raise ImportError( + "param.DataFrameLike requires the optional 'narwhals' package. " + "Install it with: pip install narwhals" + ) from e + return narwhals diff --git a/param/parameters.py b/param/parameters.py index b44da28f3..67874f60c 100644 --- a/param/parameters.py +++ b/param/parameters.py @@ -45,6 +45,7 @@ _find_stack_level, _validate_error_prefix, _deserialize_from_path, + _get_narwhals, _named_objs, _produce_value, _get_min_max_value, @@ -114,6 +115,12 @@ class _DataFrameInitKwargs(_ParameterKwargs, total=False): columns: int | tuple[int | None, int | None] | list[str] | set[str] | None ordered: bool | None + class _DataFrameLikeInitKwargs(_ParameterKwargs, total=False): + rows: int | tuple[int | None, int | None] | None + columns: int | tuple[int | None, int | None] | list[str] | set[str] | None + ordered: bool | None + eager_only: bool | None + class _SeriesInitKwargs(_ParameterKwargs, total=False): rows: int | tuple[int | None, int | None] | None @@ -3311,6 +3318,25 @@ def deserialize(cls, value): return numpy.asarray(value) +def _length_bounds_check(parameter, bounds, length, name): + """Check ``length`` against an int or ``(lower, upper)`` ``bounds``. + + Shared by :class:`DataFrame` and :class:`DataFrameLike`; ``parameter`` is + only used for the error-message prefix. + """ + message = f'{name} length {length} does not match declared bounds of {bounds}' + if not isinstance(bounds, tuple): + if (bounds != length): + raise ValueError(f"{_validate_error_prefix(parameter)}: {message}") + else: + return + (lower, upper) = bounds + failure = ((lower is not None and (length < lower)) + or (upper is not None and length > upper)) + if failure: + raise ValueError(f"{_validate_error_prefix(parameter)}: {message}") + + class DataFrame(ClassSelector["DF"]): """ Parameter whose value is a pandas ``DataFrame``. @@ -3406,17 +3432,7 @@ def __init__( self._validate(self.default) def _length_bounds_check(self, bounds, length, name): - message = f'{name} length {length} does not match declared bounds of {bounds}' - if not isinstance(bounds, tuple): - if (bounds != length): - raise ValueError(f"{_validate_error_prefix(self)}: {message}") - else: - return - (lower, upper) = bounds - failure = ((lower is not None and (length < lower)) - or (upper is not None and length > upper)) - if failure: - raise ValueError(f"{_validate_error_prefix(self)}: {message}") + _length_bounds_check(self, bounds, length, name) def _validate(self, val): super()._validate(val) @@ -3487,6 +3503,208 @@ def deserialize(cls, value): return pandas.DataFrame(value) +class DataFrameLike(ClassSelector[t.Any]): + """ + Parameter whose value is any dataframe-like object that Narwhals recognises. + + Unlike :class:`DataFrame`, which is restricted to ``pandas.DataFrame``, + ``DataFrameLike`` accepts any object supported by + `Narwhals `_ (pandas, Polars, + PyArrow, ...). The native value is passed through unchanged; authors who + want a backend-agnostic API can call ``narwhals.from_native`` themselves. + + ``rows``: number or ``(lower, upper)`` bounds on row count. + + ``columns``: number, ``(lower, upper)`` bounds, a list (exact columns, + same order unless ``ordered=False``), or a set (required subset). + + ``eager_only``: when ``True`` (default), reject lazy frames. Set + ``eager_only=False`` to also accept lazy frames (Polars ``LazyFrame``, + Dask, DuckDB); row counts on lazy frames are validated through a scalar + ``count()`` collect rather than materialising the frame. + + Serialization emits a list of records via Narwhals; ``deserialize`` + reconstructs a ``pandas.DataFrame`` because JSON carries no backend. + """ + + __slots__ = ['rows', 'columns', 'ordered', 'eager_only'] + + _slot_defaults = { + **ClassSelector._slot_defaults, + 'rows': None, + 'columns': None, + 'ordered': None, + 'eager_only': True, + } + + if t.TYPE_CHECKING: + + @t.overload + def __init__( + self: DataFrameLike, + default: t.Any = None, + *, + allow_None: t.Literal[False] = False, + doc: str | None = None, + label: str | None = None, + precedence: float | None = None, + instantiate: bool = True, + constant: bool = False, + readonly: bool = False, + pickle_default_value: bool = True, + per_instance: bool = True, + allow_refs: bool = False, + nested_refs: bool = False, + default_factory: t.Callable[[], t.Any] | None = None, + metadata: dict[str, t.Any] | None = None, + ) -> None: + ... + + @t.overload + def __init__( + self: DataFrameLike, + default: t.Any | None = None, + *, + allow_None: t.Literal[True] = True, + **kwargs: Unpack[_DataFrameLikeInitKwargs] + ) -> None: + ... + + @t.overload + def __init__( + self: DataFrameLike, + default: None = None, + *, + allow_None: t.Literal[False] = False, + **kwargs: Unpack[_DataFrameLikeInitKwargs] + ) -> None: + ... + + def __init__( + self, + default: t.Any | None = t.cast("t.Any | None", Undefined), # pyrefly: ignore[bad-argument-type] + *, + rows: int | tuple[int | None, int | None] | None = t.cast("int | tuple[int | None, int | None] | None", Undefined), # pyrefly: ignore[bad-argument-type] + columns: int | tuple[int | None, int | None] | list[str] | set[str] | None = t.cast("int | tuple[int | None, int | None] | list[str] | set[str] | None", Undefined), # pyrefly: ignore[bad-argument-type] + ordered: bool | None = t.cast("bool | None", Undefined), # pyrefly: ignore[bad-argument-type] + eager_only: bool | None = t.cast("bool | None", Undefined), # pyrefly: ignore[bad-argument-type] + allow_None: bool = t.cast("bool", Undefined), # pyrefly: ignore[bad-argument-type] + **params: Unpack[_ParameterKwargs] + ) -> None: + _get_narwhals() + object.__setattr__(self, 'rows', rows) + object.__setattr__(self, 'columns', columns) + object.__setattr__(self, 'ordered', ordered) + object.__setattr__(self, 'eager_only', eager_only) + super().__init__( # type: ignore[misc, call-overload] + default=default, # type: ignore[arg-type] + class_=object, # type: ignore[arg-type] + is_instance=True, + allow_None=allow_None, # type: ignore[arg-type] + **params, + ) + self._validate(self.default) + + def _as_narwhals(self, val): + narwhals = _get_narwhals() + try: + return narwhals.from_native( + val, eager_only=self.eager_only, pass_through=False + ) + except TypeError as e: + kind = 'an eager dataframe-like' if self.eager_only else 'a dataframe-like' + raise ValueError( + f"{_validate_error_prefix(self)} value must be {kind} object " + f"that Narwhals recognises (pandas, Polars, PyArrow, ...), " + f"not {type(val).__name__!r}." + ) from e + + def _validate(self, val): + super()._validate(val) + + if isinstance(self.columns, set) and self.ordered is True: + raise ValueError( + f'{_validate_error_prefix(self)}: columns cannot be ordered ' + f'when specified as a set' + ) + + if val is None: + # class_=object means ClassSelector accepts None even when + # allow_None is False, so reject it explicitly here. + if self.allow_None: + return + raise ValueError( + f"{_validate_error_prefix(self)} value must be a dataframe-like " + f"object that Narwhals recognises, not None." + ) + + nwframe = self._as_narwhals(val) + narwhals = _get_narwhals() + is_lazy = isinstance(nwframe, narwhals.LazyFrame) + + # Resolve schema once if any column check or lazy row check needs it. + need_schema = self.columns is not None or (self.rows is not None and is_lazy) + schema = nwframe.collect_schema() if need_schema else None + + if self.columns is not None: + assert schema is not None + cols = list(schema.names()) + if (isinstance(self.columns, tuple) and len(self.columns)==2 + and all(isinstance(v, (type(None), numbers.Number)) for v in self.columns)): # Numeric bounds tuple + _length_bounds_check(self, self.columns, len(cols), 'columns') + elif isinstance(self.columns, (list, set)): + self.ordered = isinstance(self.columns, list) if self.ordered is None else self.ordered + difference = set(self.columns) - {str(el) for el in cols} + if difference: + raise ValueError( + f"{_validate_error_prefix(self)}: provided columns " + f"{cols} does not contain required " + f"columns {sorted(self.columns)}" + ) + else: + _length_bounds_check(self, self.columns, len(cols), 'column') + + if self.ordered and isinstance(self.columns, Iterable): + if cols != list(self.columns): + raise ValueError( + f"{_validate_error_prefix(self)}: provided columns " + f"{cols} must exactly match {self.columns}" + ) + + if self.rows is not None: + if is_lazy: + assert schema is not None + first = next(iter(schema.names()), None) + n = ( + nwframe.select(narwhals.col(first).count()).collect().item() + if first is not None else 0 + ) + else: + n = nwframe.shape[0] + _length_bounds_check(self, self.rows, n, 'row') + + @classmethod + def serialize(cls, value): + # Backend-neutral list-of-records via Narwhals, so JSON output does + # not depend on the original library. A lazy frame must be collected + # here (unlike validation) because a computation graph cannot be + # serialized. + if value is None: + return None + narwhals = _get_narwhals() + nwframe = narwhals.from_native(value) + if isinstance(nwframe, narwhals.LazyFrame): + nwframe = nwframe.collect() + return nwframe.rows(named=True) + + @classmethod + def deserialize(cls, value): + # JSON carries no backend information, so deserialization lands on + # pandas (the universal default), exactly like DataFrame. Callers + # needing another backend can reconstruct from the records form. + return DataFrame.deserialize(value) + + class Series(ClassSelector["ST"]): """ Parameter whose value is a pandas ``Series``. diff --git a/pixi.toml b/pixi.toml index a62a99629..72c880a1a 100644 --- a/pixi.toml +++ b/pixi.toml @@ -130,11 +130,13 @@ test-unit = 'pytest tests' cloudpickle = "*" ipython = "*" jsonschema = "*" +narwhals = "*" nest-asyncio = "*" numpy = "*" odfpy = "*" openpyxl = "*" pandas = "*" +polars = "*" pyarrow = "*" pytables = "*" xlrd = "*" @@ -207,8 +209,10 @@ lint-install = 'pre-commit install' [feature.type.dependencies] ty = "==0.0.34" mypy = "*" +narwhals = "*" numpy = "*" pandas = "*" +polars = "*" pyrefly = "*" pyright = "*" IPython = "*" diff --git a/pyproject.toml b/pyproject.toml index 6f91c5f24..b7a3ba9f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,8 @@ tests-full = [ "param[tests-deser]", "numpy", "pandas", + "narwhals", + "polars", "ipython", "jsonschema", "gmpy2", diff --git a/tests/testdataframelike.py b/tests/testdataframelike.py new file mode 100644 index 000000000..c21648650 --- /dev/null +++ b/tests/testdataframelike.py @@ -0,0 +1,246 @@ +"""Test the DataFrameLike Parameter (cross-backend via Narwhals).""" +import pytest + +import param + +from .utils import check_defaults + +pytest.importorskip("narwhals") + +try: + import pandas as pd +except ModuleNotFoundError: + pd = None + +try: + import polars as pl +except ModuleNotFoundError: + pl = None + +try: + import pyarrow as pa +except ModuleNotFoundError: + pa = None + +skip_no_pandas = pytest.mark.skipif(pd is None, reason="pandas not available") +skip_no_polars = pytest.mark.skipif(pl is None, reason="polars not available") +skip_no_pyarrow = pytest.mark.skipif(pa is None, reason="pyarrow not available") + + +def test_defaults_class(): + class P(param.Parameterized): + df = param.DataFrameLike() + + check_defaults(P.param.df, label='Df', skip=['instantiate']) + + +def test_defaults_inst(): + class P(param.Parameterized): + df = param.DataFrameLike() + + check_defaults(P().param.df, label='Df', skip=['instantiate']) + + +@skip_no_pandas +def test_accepts_pandas(): + class P(param.Parameterized): + df = param.DataFrameLike(default=pd.DataFrame({'a': [1, 2]})) + src = pd.DataFrame({'a': [3, 4]}) + p = P(df=src) + # Value is passed through unchanged (no Narwhals wrapper). + assert p.df is src + + +@skip_no_pandas +@skip_no_polars +def test_accepts_polars_eager(): + class P(param.Parameterized): + df = param.DataFrameLike(default=pd.DataFrame({'a': [1]})) + src = pl.DataFrame({'a': [1, 2]}) + p = P(df=src) + assert p.df is src + assert isinstance(p.df, pl.DataFrame) + + +@skip_no_pandas +@skip_no_pyarrow +def test_accepts_pyarrow(): + class P(param.Parameterized): + df = param.DataFrameLike(default=pd.DataFrame({'a': [1]})) + src = pa.table({'a': [1, 2]}) + assert P(df=src).df is src + + +@pytest.fixture +def P_pandas(): + pytest.importorskip("pandas") + + class P(param.Parameterized): + df = param.DataFrameLike(default=pd.DataFrame({'a': [1]})) + + return P + + +@skip_no_pandas +class TestDataFrameLikeRejects: + + def test_list(self, P_pandas): + with pytest.raises(ValueError): + P_pandas(df=[1, 2, 3]) + + def test_dict(self, P_pandas): + with pytest.raises(ValueError): + P_pandas(df={'a': [1]}) + + def test_str(self, P_pandas): + with pytest.raises(ValueError): + P_pandas(df='not a frame') + + def test_series(self, P_pandas): + with pytest.raises(ValueError): + P_pandas(df=pd.Series([1, 2, 3])) + + def test_none_without_allow_none(self, P_pandas): + with pytest.raises(ValueError): + P_pandas(df=None) + + def test_none_with_allow_none(self): + class Q(param.Parameterized): + df = param.DataFrameLike(default=None, allow_None=True) + assert Q(df=None).df is None + + +@skip_no_pandas +class TestDataFrameLikeRows: + + def test_rows_exact_ok(self): + class P(param.Parameterized): + df = param.DataFrameLike(default=pd.DataFrame({'a': [1, 2, 3]}), rows=3) + P(df=pd.DataFrame({'a': [4, 5, 6]})) + + def test_rows_exact_mismatch(self): + class P(param.Parameterized): + df = param.DataFrameLike(default=pd.DataFrame({'a': [1, 2, 3]}), rows=3) + with pytest.raises(ValueError): + P(df=pd.DataFrame({'a': [1, 2]})) + + def test_rows_bounds(self): + class P(param.Parameterized): + df = param.DataFrameLike(default=pd.DataFrame({'a': [1, 2]}), rows=(1, 4)) + P(df=pd.DataFrame({'a': [1, 2, 3, 4]})) + with pytest.raises(ValueError): + P(df=pd.DataFrame({'a': list(range(5))})) + + @skip_no_polars + def test_rows_polars(self): + class P(param.Parameterized): + df = param.DataFrameLike(default=pd.DataFrame({'a': [1, 2]}), rows=2) + P(df=pl.DataFrame({'a': [9, 8]})) + with pytest.raises(ValueError): + P(df=pl.DataFrame({'a': [1]})) + + +@skip_no_pandas +class TestDataFrameLikeColumns: + + def test_set_subset_ok(self): + class P(param.Parameterized): + df = param.DataFrameLike( + default=pd.DataFrame({'a': [1], 'b': [2]}), columns={'a'}) + P(df=pd.DataFrame({'a': [1], 'b': [2], 'c': [3]})) + + def test_set_missing_column(self): + class P(param.Parameterized): + df = param.DataFrameLike( + default=pd.DataFrame({'a': [1]}), columns={'a'}) + with pytest.raises(ValueError): + P(df=pd.DataFrame({'x': [1]})) + + def test_list_exact_ordered(self): + class P(param.Parameterized): + df = param.DataFrameLike( + default=pd.DataFrame({'a': [1], 'b': [2]}), columns=['a', 'b']) + P(df=pd.DataFrame({'a': [1], 'b': [2]})) + with pytest.raises(ValueError): + P(df=pd.DataFrame({'b': [2], 'a': [1]})) + + def test_columns_numeric_bounds(self): + class P(param.Parameterized): + df = param.DataFrameLike( + default=pd.DataFrame({'a': [1], 'b': [2]}), columns=(1, 3)) + P(df=pd.DataFrame({'a': [1], 'b': [2], 'c': [3]})) + with pytest.raises(ValueError): + P(df=pd.DataFrame({c: [1] for c in 'abcd'})) + + def test_set_with_ordered_raises(self): + with pytest.raises(ValueError): + class P(param.Parameterized): + df = param.DataFrameLike( + default=pd.DataFrame({'a': [1]}), + columns={'a'}, ordered=True) + + +@skip_no_polars +class TestDataFrameLikeLazy: + + def test_lazy_rejected_by_default(self): + class P(param.Parameterized): + df = param.DataFrameLike(default=pl.DataFrame({'a': [1]})) + with pytest.raises(ValueError): + P(df=pl.LazyFrame({'a': [1, 2]})) + + def test_lazy_accepted_when_allowed(self): + class P(param.Parameterized): + df = param.DataFrameLike( + default=pl.DataFrame({'a': [1]}), eager_only=False) + src = pl.LazyFrame({'a': [1, 2]}) + assert P(df=src).df is src + + def test_lazy_columns_still_validated(self): + class P(param.Parameterized): + df = param.DataFrameLike( + default=pl.DataFrame({'a': [1]}), + columns={'a'}, eager_only=False) + P(df=pl.LazyFrame({'a': [1], 'b': [2]})) + with pytest.raises(ValueError): + P(df=pl.LazyFrame({'x': [1]})) + + def test_lazy_rows_validated_via_count(self): + # rows=2 must be validated against a LazyFrame without materialising + # the whole frame; narwhals .count() is used to pull only a scalar. + class P(param.Parameterized): + df = param.DataFrameLike( + default=pl.DataFrame({'a': [1, 2]}), + rows=2, eager_only=False) + # Matching row count passes. + P(df=pl.LazyFrame({'a': [1, 2]})) + # Non-matching row count fails (proves rows are actually checked). + with pytest.raises(ValueError): + P(df=pl.LazyFrame({'a': [1, 2, 3]})) + + +@skip_no_pandas +class TestDataFrameLikeSerialize: + + def test_serialize_none(self): + assert param.DataFrameLike.serialize(None) is None + + def test_serialize_records(self): + recs = param.DataFrameLike.serialize(pd.DataFrame({'a': [1, 2], 'b': ['x', 'y']})) + assert recs == [{'a': 1, 'b': 'x'}, {'a': 2, 'b': 'y'}] + + @skip_no_polars + def test_serialize_backend_neutral(self): + recs_pd = param.DataFrameLike.serialize(pd.DataFrame({'a': [1, 2]})) + recs_pl = param.DataFrameLike.serialize(pl.DataFrame({'a': [1, 2]})) + assert recs_pd == recs_pl + + @skip_no_polars + def test_serialize_lazy_collected(self): + recs = param.DataFrameLike.serialize(pl.LazyFrame({'a': [1, 2]})) + assert recs == [{'a': 1}, {'a': 2}] + + def test_deserialize_roundtrip(self): + recs = param.DataFrameLike.serialize(pd.DataFrame({'a': [1, 2]})) + back = param.DataFrameLike.deserialize(recs) + assert back.to_dict('records') == recs diff --git a/tests/testdefaults.py b/tests/testdefaults.py index 7adbb7c7d..63ae13434 100644 --- a/tests/testdefaults.py +++ b/tests/testdefaults.py @@ -28,6 +28,10 @@ except ModuleNotFoundError: skip.append('DataFrame') skip.append('Series') +try: + import narwhals # noqa +except ModuleNotFoundError: + skip.append('DataFrameLike') class DefaultsMetaclassTest(type):