diff --git a/docs/extending.md b/docs/extending.md index 0ddb1e39f7..4896b9569b 100644 --- a/docs/extending.md +++ b/docs/extending.md @@ -59,7 +59,40 @@ handle plugins. For this integration to work, any plugin architecture must conta Take a look at the `Plugin` protocol in `narwhals/plugins.py` for the signatures. - + +### Supporting `backend=...` in Narwhals functions + +Functions and constructors which accept a `backend` argument can also dispatch to a +plugin. Users can pass: + +- the plugin's entry point name (e.g. `backend="narwhals-grizzlies"`), +- the plugin's module name (e.g. `backend="narwhals_grizzlies"`), +- or the plugin's module itself (e.g. `backend=narwhals_grizzlies`). + +There are two mechanisms, depending on the function: + +1. **IO functions** (`read_csv`, `scan_csv`, `read_parquet`, `scan_parquet`): the plugin + should expose a same-named function at the top level of its namespace which returns a + **native** object (which is then passed through `narwhals.from_native`): + + - `read_csv(source, separator=",", **kwargs)`, `scan_csv(source, separator=",", **kwargs)` + - `read_parquet(source, **kwargs)`, `scan_parquet(source, **kwargs)` + + If the required hook is missing, an informative `AttributeError` is raised. + +2. **Eager constructors** (`from_dict`, `from_dicts`, `from_numpy`, `from_arrow`, + `new_series`, as well as the `DataFrame.from_*` and `Series.from_*` classmethods): + these are eager-only, and go through the compliant namespace returned by the plugin's + `__narwhals_namespace__`. If that namespace implements the `EagerNamespace` protocol + (in particular the `_dataframe` and `_series` properties, and the `from_dict`, + `from_dicts`, `from_numpy`, `from_arrow` and `from_iterable` constructors on the + respective compliant classes), these functions work with no extra plugin code. + Lazy-only plugins get an informative `ValueError` instead. + +Methods which internally construct Series (for example `Series.scatter`, or +`DataFrame.filter` with a list of booleans) use the compliant namespace of the object +they are called on, so they also work for eager plugins. + ## Can I see an example? Yes! For a reference plugin, please check out [narwhals-daft](https://github.com/narwhals-dev/narwhals-daft). diff --git a/packages/test-plugin/src/test_plugin/__init__.py b/packages/test-plugin/src/test_plugin/__init__.py index 15074b5c27..36e884f021 100644 --- a/packages/test-plugin/src/test_plugin/__init__.py +++ b/packages/test-plugin/src/test_plugin/__init__.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from typing_extensions import TypeIs @@ -21,3 +21,30 @@ def is_native(native_object: object) -> TypeIs[DictFrame]: NATIVE_PACKAGE = "builtins" + + +# The functions below are the IO extension hooks used by `narwhals.functions` +# when `backend` resolves to this plugin (https://github.com/narwhals-dev/narwhals/issues/3713). +# Eager constructors (`from_dict`, `new_series`, `Series.from_iterable`, ...) instead go +# through `DictNamespace._dataframe` / `DictNamespace._series`. + + +def scan_csv(source: str, separator: str = ",", **kwargs: Any) -> DictFrame: # noqa: ARG001 + import csv + from pathlib import Path + + with Path(source).open(newline="", encoding="utf-8") as file: + header, *rows = list(csv.reader(file, delimiter=separator)) + return {name: [row[index] for row in rows] for index, name in enumerate(header)} + + +def scan_parquet(source: str, **kwargs: Any) -> DictFrame: + import pyarrow.parquet as pq + + result: DictFrame = pq.read_table(source, **kwargs).to_pydict() + return result + + +# `read_*` returns the same native object; narwhals then enforces eagerness downstream. +read_csv = scan_csv +read_parquet = scan_parquet diff --git a/packages/test-plugin/src/test_plugin/dataframe.py b/packages/test-plugin/src/test_plugin/dataframe.py index 56bc928cf7..d429e0630a 100644 --- a/packages/test-plugin/src/test_plugin/dataframe.py +++ b/packages/test-plugin/src/test_plugin/dataframe.py @@ -11,15 +11,91 @@ from narwhals.typing import CompliantLazyFrame if TYPE_CHECKING: + from collections.abc import Mapping, Sequence from typing import TypeAlias from typing_extensions import Self - from narwhals import LazyFrame # noqa: F401 + from narwhals import DataFrame, LazyFrame # noqa: F401 + from narwhals._utils import _LimitedContext DictFrame: TypeAlias = dict[str, list[Any]] +class DictDataFrame: + """Minimal eager frame, kept to the smallest surface exercised in narwhals' tests.""" + + _implementation = Implementation.UNKNOWN + + def __init__(self, native_dataframe: DictFrame, *, version: Version) -> None: + self._native_frame: DictFrame = native_dataframe + self._version = version + + @classmethod + def from_dict( + cls, + data: Mapping[str, Any], + /, + *, + context: _LimitedContext, + schema: Any = None, # noqa: ARG003 + ) -> Self: + return cls( + {name: list(values) for name, values in data.items()}, + version=context._version, + ) + + @classmethod + def from_dicts( + cls, + data: Sequence[Mapping[str, Any]], + /, + *, + context: _LimitedContext, + schema: Any = None, # noqa: ARG003 + ) -> Self: + columns: list[str] = list(data[0]) if data else [] + return cls( + {name: [row[name] for row in data] for name in columns}, + version=context._version, + ) + + @classmethod + def from_numpy( + cls, data: Any, /, *, context: _LimitedContext, schema: Any = None + ) -> Self: + names = ( + list(schema) + if schema is not None + else [str(index) for index in range(data.shape[1])] + ) + return cls( + {name: data[:, index].tolist() for index, name in enumerate(names)}, + version=context._version, + ) + + @classmethod + def from_arrow(cls, data: Any, /, *, context: _LimitedContext) -> Self: + import pyarrow as pa + + return cls(pa.table(data).to_pydict(), version=context._version) + + def __narwhals_dataframe__(self) -> Self: + return self + + def __narwhals_namespace__(self) -> Any: + from test_plugin.namespace import DictNamespace + + return DictNamespace(version=self._version) + + @property + def native(self) -> DictFrame: + return self._native_frame + + def to_narwhals(self) -> DataFrame[Any]: + return self._version.dataframe(self, level="full") + + class DictLazyFrame( CompliantLazyFrame[Any, "DictFrame", "LazyFrame[DictFrame]"], # type: ignore[type-var] ValidateBackendVersion, diff --git a/packages/test-plugin/src/test_plugin/namespace.py b/packages/test-plugin/src/test_plugin/namespace.py index 3eba7c3434..bb7085593f 100644 --- a/packages/test-plugin/src/test_plugin/namespace.py +++ b/packages/test-plugin/src/test_plugin/namespace.py @@ -3,11 +3,12 @@ from typing import TYPE_CHECKING, Any from narwhals._compliant import CompliantNamespace -from narwhals._utils import not_implemented -from test_plugin.dataframe import DictFrame, DictLazyFrame +from narwhals._utils import Implementation, not_implemented +from test_plugin.dataframe import DictDataFrame, DictFrame, DictLazyFrame if TYPE_CHECKING: from narwhals.utils import Version + from test_plugin.series import DictSeries class DictNamespace(CompliantNamespace[DictLazyFrame, Any]): @@ -17,9 +18,22 @@ def __init__(self, *, version: Version) -> None: def from_native(self, native_object: DictFrame) -> DictLazyFrame: return DictLazyFrame(native_object, version=self._version) + @property + def _dataframe(self) -> type[DictDataFrame]: + return DictDataFrame + + @property + def _series(self) -> type[DictSeries]: + from test_plugin.series import DictSeries + + return DictSeries + + # NOTE: `not_implemented.__get__` reads `instance._implementation` to build its + # error message, so `_implementation` itself must be a real value. + _implementation = Implementation.UNKNOWN + is_native: Any = not_implemented() _expr: Any = not_implemented() - _implementation: Any = not_implemented() corr: Any = not_implemented() cov: Any = not_implemented() len: Any = not_implemented() diff --git a/packages/test-plugin/src/test_plugin/series.py b/packages/test-plugin/src/test_plugin/series.py new file mode 100644 index 0000000000..6fc82d076e --- /dev/null +++ b/packages/test-plugin/src/test_plugin/series.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from narwhals._utils import Implementation + +if TYPE_CHECKING: + from collections.abc import Iterable + + from typing_extensions import Self + + from narwhals._utils import Version, _LimitedContext + from narwhals.series import Series + + +class DictSeries: + """Minimal eager series, kept to the smallest surface exercised in narwhals' tests.""" + + _implementation = Implementation.UNKNOWN + + def __init__( + self, values: Iterable[Any], *, name: str = "", version: Version + ) -> None: + self._values: list[Any] = list(values) + self._name = name + self._version = version + + @classmethod + def from_iterable( + cls, + data: Iterable[Any], + /, + *, + context: _LimitedContext, + name: str = "", + dtype: Any = None, # noqa: ARG003 + ) -> Self: + return cls(data, name=name, version=context._version) + + @classmethod + def from_numpy(cls, data: Any, /, *, context: _LimitedContext) -> Self: + return cls(data.tolist(), version=context._version) + + def __narwhals_series__(self) -> Self: + return self + + def __narwhals_namespace__(self) -> Any: + from test_plugin.namespace import DictNamespace + + return DictNamespace(version=self._version) + + @property + def native(self) -> list[Any]: + return self._values + + @property + def name(self) -> str: + return self._name + + def alias(self, name: str) -> Self: + return self.__class__(self._values, name=name, version=self._version) + + def is_empty(self) -> bool: + return not self._values + + def scatter(self, indices: Self, values: Self) -> Self: + data = list(self._values) + for index, value in zip(indices.native, values.native, strict=True): + data[index] = value + return self.__class__(data, name=self._name, version=self._version) + + def to_narwhals(self) -> Series[Any]: + return self._version.series(self, level="full") diff --git a/src/narwhals/_utils.py b/src/narwhals/_utils.py index 4bb942f91a..b996af813d 100644 --- a/src/narwhals/_utils.py +++ b/src/narwhals/_utils.py @@ -77,6 +77,7 @@ from narwhals._compliant.any_namespace import NamespaceAccessor from narwhals._compliant.typing import ( Accessor, + EagerNamespaceAny, EvalNames, NativeDataFrameT, NativeLazyFrameT, @@ -1625,6 +1626,82 @@ def is_eager_allowed(impl: Implementation, /) -> TypeIs[_EagerAllowedImpl]: } +def _ensure_eager_capable( + namespace: Any, /, *, source: str, function_name: str +) -> EagerNamespaceAny: + """Duck-check that `namespace` implements the `EagerNamespace` protocol. + + `_series` and `_dataframe` may be regular properties, missing entirely, or + `not_implemented` descriptors (which raise on instance access). + """ + try: + eager_capable = namespace._series is not None and namespace._dataframe is not None + except (AttributeError, NotImplementedError): + eager_capable = False + if not eager_capable: + msg = ( + f"Plugin backend {source!r} does not provide eager support (its " + "compliant namespace does not implement the `EagerNamespace` protocol), " + f"but `{function_name}` is an eager-only function." + ) + raise ValueError(msg) + return cast("EagerNamespaceAny", namespace) + + +def eager_namespace( + backend: IntoBackend[Backend], + /, + *, + version: Version, + function_name: str, + hint_example: str, +) -> EagerNamespaceAny: + """Resolve `backend` to an eager-capable compliant namespace. + + Built-in eager backends resolve directly. Anything unknown to `Implementation` is + resolved via the plugin entry-point registry, in which case the plugin's + `__narwhals_namespace__` must return a namespace implementing the `EagerNamespace` + protocol (in particular, the `_series` and `_dataframe` properties). + Built-in lazy-only backends raise an informative `ValueError`, suggesting + `hint_example` followed by a `.lazy(...)` call. + """ + implementation = Implementation.from_backend(backend) + if is_eager_allowed(implementation): + # NOTE: `cast` is required as the overload returns a union of concrete + # namespaces, whose invariant type parameters don't unify with `*Any` aliases. + return cast( + "EagerNamespaceAny", version.namespace.from_backend(implementation).compliant + ) + if implementation is not Implementation.UNKNOWN: + msg = ( + f"{implementation} support in Narwhals is lazy-only, but `{function_name}` is an eager-only function.\n\n" + "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" + f" {hint_example}.lazy('{implementation}')" + ) + raise ValueError(msg) + from narwhals.plugins import _backend_namespace, _plugin_hook + + module = _backend_namespace(backend) + namespace = _plugin_hook(module, "__narwhals_namespace__")(version=version) + return _ensure_eager_capable( + namespace, source=module.__name__, function_name=function_name + ) + + +def eager_namespace_from_compliant( + compliant_object: Any, /, *, function_name: str +) -> EagerNamespaceAny: + """Resolve the eager namespace of a compliant object originating from a plugin. + + `Implementation.UNKNOWN` cannot be resolved back to a plugin, so methods which + internally construct series use the namespace of the compliant object itself. + """ + namespace = compliant_object.__narwhals_namespace__() + return _ensure_eager_capable( + namespace, source=type(namespace).__name__, function_name=function_name + ) + + def can_lazyframe_collect(impl: Implementation, /) -> TypeIs[_LazyFrameCollectImpl]: """Return True if `LazyFrame.collect(impl)` is allowed.""" return impl in {Implementation.PANDAS, Implementation.POLARS, Implementation.PYARROW} diff --git a/src/narwhals/dataframe.py b/src/narwhals/dataframe.py index adc8056cf0..3743f8ca42 100644 --- a/src/narwhals/dataframe.py +++ b/src/narwhals/dataframe.py @@ -29,11 +29,12 @@ _resolve_sample_size, can_lazyframe_collect, check_columns_exist, + eager_namespace, + eager_namespace_from_compliant, flatten, generate_repr, is_compliant_dataframe, is_compliant_lazyframe, - is_eager_allowed, is_index_selector, is_iterator, is_lazy_allowed, @@ -540,17 +541,14 @@ def from_arrow( if not (supports_arrow_c_stream(native_frame) or is_pyarrow_table(native_frame)): msg = f"Given object of type {type(native_frame)} does not support PyCapsule interface" raise TypeError(msg) - implementation = Implementation.from_backend(backend) - if is_eager_allowed(implementation): - ns = cls._version.namespace.from_backend(implementation).compliant - compliant = ns._dataframe.from_arrow(native_frame, context=ns) - return cls(compliant, level="full") - msg = ( - f"{implementation} support in Narwhals is lazy-only, but `DataFrame.from_arrow` is an eager-only function.\n\n" - "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" - f" nw.DataFrame.from_arrow(df, backend='pyarrow').lazy('{implementation}')" + ns = eager_namespace( + backend, + version=cls._version, + function_name="DataFrame.from_arrow", + hint_example="nw.DataFrame.from_arrow(df, backend='pyarrow')", ) - raise ValueError(msg) + compliant = ns._dataframe.from_arrow(native_frame, context=ns) + return cls(compliant, level="full") @classmethod def from_dict( @@ -600,18 +598,14 @@ def from_dict( """ if backend is None: data, backend = _from_dict_no_backend(data) - implementation = Implementation.from_backend(backend) - if is_eager_allowed(implementation): - ns = cls._version.namespace.from_backend(implementation).compliant - compliant = ns._dataframe.from_dict(data, schema=schema, context=ns) - return cls(compliant, level="full") - # NOTE: (#2786) needs resolving for extensions - msg = ( - f"{implementation} support in Narwhals is lazy-only, but `DataFrame.from_dict` is an eager-only function.\n\n" - "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" - f" nw.DataFrame.from_dict({{'a': [1, 2]}}, backend='pyarrow').lazy('{implementation}')" + ns = eager_namespace( + backend, + version=cls._version, + function_name="DataFrame.from_dict", + hint_example="nw.DataFrame.from_dict({'a': [1, 2]}, backend='pyarrow')", ) - raise ValueError(msg) + compliant = ns._dataframe.from_dict(data, schema=schema, context=ns) + return cls(compliant, level="full") @classmethod def from_dicts( @@ -672,18 +666,14 @@ def from_dicts( |└───────┴────────┴───────┘| └──────────────────────────┘ """ - implementation = Implementation.from_backend(backend) - if is_eager_allowed(implementation): - ns = cls._version.namespace.from_backend(implementation).compliant - compliant = ns._dataframe.from_dicts(data, schema=schema, context=ns) - return cls(compliant, level="full") - # NOTE: (#2786) needs resolving for extensions - msg = ( - f"{implementation} support in Narwhals is lazy-only, but `DataFrame.from_dicts` is an eager-only function.\n\n" - "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" - f" nw.DataFrame.from_dicts([{{'a': 1}}, {{'a': 2}}], backend='pyarrow').lazy('{implementation}')" + ns = eager_namespace( + backend, + version=cls._version, + function_name="DataFrame.from_dicts", + hint_example="nw.DataFrame.from_dicts([{'a': 1}, {'a': 2}], backend='pyarrow')", ) - raise ValueError(msg) + compliant = ns._dataframe.from_dicts(data, schema=schema, context=ns) + return cls(compliant, level="full") @classmethod def from_numpy( @@ -745,16 +735,14 @@ def from_numpy( f"Got {type(schema)}." ) raise TypeError(msg) - implementation = Implementation.from_backend(backend) - if is_eager_allowed(implementation): - ns = cls._version.namespace.from_backend(implementation).compliant - return cls(ns.from_numpy(data, schema), level="full") - msg = ( - f"{implementation} support in Narwhals is lazy-only, but `DataFrame.from_numpy` is an eager-only function.\n\n" - "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" - f" nw.DataFrame.from_numpy(arr, backend='pyarrow').lazy('{implementation}')" + ns = eager_namespace( + backend, + version=cls._version, + function_name="DataFrame.from_numpy", + hint_example="nw.DataFrame.from_numpy(arr, backend='pyarrow')", ) - raise ValueError(msg) + compliant = ns._dataframe.from_numpy(data, schema=schema, context=ns) + return cls(compliant, level="full") def __len__(self) -> int: return self._compliant_frame.__len__() @@ -1712,9 +1700,19 @@ def filter( 1 2 7 b """ impl = self.implementation + + def into_series(values: list[bool]) -> Series[Any]: + if impl is Implementation.UNKNOWN: # type: ignore[comparison-overlap] + ns = eager_namespace_from_compliant( + self._compliant_frame, function_name="DataFrame.filter(list[bool])" + ) + return self._series( + ns._series.from_iterable(values, context=ns, name=""), level="full" + ) + return self._series.from_iterable("", values, backend=impl) + parsed_predicates = ( - self._series.from_iterable("", p, backend=impl) if is_list_of(p, bool) else p - for p in predicates + into_series(p) if is_list_of(p, bool) else p for p in predicates ) return super().filter(*parsed_predicates, **constraints) diff --git a/src/narwhals/functions.py b/src/narwhals/functions.py index 7f2151083e..2434df3c9f 100644 --- a/src/narwhals/functions.py +++ b/src/narwhals/functions.py @@ -16,8 +16,8 @@ Implementation, Version, deprecate_native_namespace, + eager_namespace, flatten, - is_eager_allowed, is_nested_literal, is_sequence_but_not_str, normalize_path, @@ -32,6 +32,7 @@ ) from narwhals.exceptions import InvalidOperationError from narwhals.expr import Expr +from narwhals.plugins import _backend_namespace, _plugin_hook from narwhals.translate import from_native, to_native if TYPE_CHECKING: @@ -40,7 +41,7 @@ from typing_extensions import Self, TypeIs - from narwhals._native import NativeDataFrame, NativeLazyFrame, NativeSeries + from narwhals._native import NativeDataFrame, NativeLazyFrame from narwhals._translate import IntoArrowTable from narwhals._typing import Backend, EagerAllowed, IntoBackend from narwhals.dataframe import DataFrame, LazyFrame @@ -213,27 +214,14 @@ def _new_series_impl( *, backend: IntoBackend[EagerAllowed], ) -> Series[Any]: - implementation = Implementation.from_backend(backend) - if is_eager_allowed(implementation): - ns = Version.MAIN.namespace.from_backend(implementation).compliant - series = ns._series.from_iterable(values, name=name, context=ns, dtype=dtype) - return series.to_narwhals() - if implementation is Implementation.UNKNOWN: # pragma: no cover - _native_namespace = implementation.to_native_namespace() - try: - native_series: NativeSeries = _native_namespace.new_series( - name, values, dtype - ) - return from_native(native_series, series_only=True).alias(name) - except AttributeError as e: - msg = "Unknown namespace is expected to implement `new_series` constructor." - raise AttributeError(msg) from e - msg = ( - f"{implementation} support in Narwhals is lazy-only, but `new_series` is an eager-only function.\n\n" - "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" - f" nw.new_series('a', [1,2,3], backend='pyarrow').to_frame().lazy('{implementation}')" + ns = eager_namespace( + backend, + version=Version.MAIN, + function_name="new_series", + hint_example="nw.new_series('a', [1,2,3], backend='pyarrow').to_frame()", ) - raise ValueError(msg) + series = ns._series.from_iterable(values, name=name, context=ns, dtype=dtype) + return series.to_narwhals() @deprecate_native_namespace(warn_version="1.26.0") @@ -288,28 +276,13 @@ def from_dict( if schema and data and (diff := set(schema.keys()).symmetric_difference(data.keys())): msg = f"Keys in `schema` and `data` are expected to match, found unmatched keys: {diff}" raise InvalidOperationError(msg) - implementation = Implementation.from_backend(backend) - if is_eager_allowed(implementation): - ns = Version.MAIN.namespace.from_backend(implementation).compliant - return ns._dataframe.from_dict(data, schema=schema, context=ns).to_narwhals() - if implementation is Implementation.UNKNOWN: # pragma: no cover - _native_namespace = implementation.to_native_namespace() - try: - # implementation is UNKNOWN, Narwhals extension using this feature should - # implement `from_dict` function in the top-level namespace. - native_frame: NativeDataFrame = _native_namespace.from_dict( - data, schema=schema - ) - except AttributeError as e: - msg = "Unknown namespace is expected to implement `from_dict` function." - raise AttributeError(msg) from e - return from_native(native_frame, eager_only=True) - msg = ( - f"{implementation} support in Narwhals is lazy-only, but `from_dict` is an eager-only function.\n\n" - "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" - f" nw.from_dict({{'a': [1, 2]}}, backend='pyarrow').lazy('{implementation}')" + ns = eager_namespace( + backend, + version=Version.MAIN, + function_name="from_dict", + hint_example="nw.from_dict({'a': [1, 2]}, backend='pyarrow')", ) - raise ValueError(msg) + return ns._dataframe.from_dict(data, schema=schema, context=ns).to_narwhals() def _from_dict_no_backend( @@ -443,28 +416,13 @@ def from_numpy( f"Got {type(schema)}." ) raise TypeError(msg) - implementation = Implementation.from_backend(backend) - if is_eager_allowed(implementation): - ns = Version.MAIN.namespace.from_backend(implementation).compliant - return ns.from_numpy(data, schema).to_narwhals() - if implementation is Implementation.UNKNOWN: # pragma: no cover - _native_namespace = implementation.to_native_namespace() - try: - # implementation is UNKNOWN, Narwhals extension using this feature should - # implement `from_numpy` function in the top-level namespace. - native_frame: NativeDataFrame = _native_namespace.from_numpy( - data, schema=schema - ) - except AttributeError as e: - msg = "Unknown namespace is expected to implement `from_numpy` function." - raise AttributeError(msg) from e - return from_native(native_frame, eager_only=True) - msg = ( - f"{implementation} support in Narwhals is lazy-only, but `from_numpy` is an eager-only function.\n\n" - "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" - f" nw.from_numpy(arr, backend='pyarrow').lazy('{implementation}')" + ns = eager_namespace( + backend, + version=Version.MAIN, + function_name="from_numpy", + hint_example="nw.from_numpy(arr, backend='pyarrow')", ) - raise ValueError(msg) + return ns._dataframe.from_numpy(data, schema=schema, context=ns).to_narwhals() def _is_into_schema(obj: Any) -> TypeIs[_IntoSchema]: @@ -515,26 +473,13 @@ def from_arrow( if not (supports_arrow_c_stream(native_frame) or is_pyarrow_table(native_frame)): msg = f"Given object of type {type(native_frame)} does not support PyCapsule interface" raise TypeError(msg) - implementation = Implementation.from_backend(backend) - if is_eager_allowed(implementation): - ns = Version.MAIN.namespace.from_backend(implementation).compliant - return ns._dataframe.from_arrow(native_frame, context=ns).to_narwhals() - if implementation is Implementation.UNKNOWN: # pragma: no cover - _native_namespace = implementation.to_native_namespace() - try: - # implementation is UNKNOWN, Narwhals extension using this feature should - # implement PyCapsule support - native: NativeDataFrame = _native_namespace.DataFrame(native_frame) - except AttributeError as e: - msg = "Unknown namespace is expected to implement `DataFrame` class which accepts object which supports PyCapsule Interface." - raise AttributeError(msg) from e - return from_native(native, eager_only=True) - msg = ( - f"{implementation} support in Narwhals is lazy-only, but `from_arrow` is an eager-only function.\n\n" - "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" - f" nw.from_arrow(df, backend='pyarrow').lazy('{implementation}')" + ns = eager_namespace( + backend, + version=Version.MAIN, + function_name="from_arrow", + hint_example="nw.from_arrow(df, backend='pyarrow')", ) - raise ValueError(msg) + return ns._dataframe.from_arrow(native_frame, context=ns).to_narwhals() def _get_sys_info() -> dict[str, str]: @@ -678,7 +623,11 @@ def read_csv( └──────────────────┘ """ impl = Implementation.from_backend(backend) - native_namespace = impl.to_native_namespace() + native_namespace = ( + _backend_namespace(backend) + if impl is Implementation.UNKNOWN + else impl.to_native_namespace() + ) native_frame: NativeDataFrame if impl in {Implementation.PANDAS, Implementation.MODIN, Implementation.CUDF}: _validate_separators(separator, ("sep",), **kwargs) @@ -707,14 +656,10 @@ def read_csv( f"Hint: use nw.scan_csv(source={source}, backend={backend})" ) raise ValueError(msg) - else: # pragma: no cover - try: - # implementation is UNKNOWN, Narwhals extension using this feature should - # implement `read_csv` function in the top-level namespace. - native_frame = native_namespace.read_csv(source=source, **kwargs) - except AttributeError as e: - msg = "Unknown namespace is expected to implement `read_csv` function." - raise AttributeError(msg) from e + else: + native_frame = _plugin_hook(native_namespace, "read_csv")( + normalize_path(source), separator=separator, **kwargs + ) return from_native(native_frame, eager_only=True) @@ -759,7 +704,11 @@ def scan_csv( └─────────┴───────┘ """ implementation = Implementation.from_backend(backend) - native_namespace = implementation.to_native_namespace() + native_namespace = ( + _backend_namespace(backend) + if implementation is Implementation.UNKNOWN + else implementation.to_native_namespace() + ) native_frame: NativeDataFrame | NativeLazyFrame source = normalize_path(source) if implementation is Implementation.POLARS: @@ -795,14 +744,10 @@ def scan_csv( ) else csv_reader.options(sep=separator, **kwargs).load(source) ) - else: # pragma: no cover - try: - # implementation is UNKNOWN, Narwhals extension using this feature should - # implement `scan_csv` function in the top-level namespace. - native_frame = native_namespace.scan_csv(source=source, **kwargs) - except AttributeError as e: - msg = "Unknown namespace is expected to implement `scan_csv` function." - raise AttributeError(msg) from e + else: + native_frame = _plugin_hook(native_namespace, "scan_csv")( + source, separator=separator, **kwargs + ) return from_native(native_frame).lazy() @@ -841,7 +786,11 @@ def read_parquet( └──────────────────┘ """ impl = Implementation.from_backend(backend) - native_namespace = impl.to_native_namespace() + native_namespace = ( + _backend_namespace(backend) + if impl is Implementation.UNKNOWN + else impl.to_native_namespace() + ) native_frame: NativeDataFrame if impl in { Implementation.POLARS, @@ -868,14 +817,10 @@ def read_parquet( f"Hint: use nw.scan_parquet(source={source}, backend={backend})" ) raise ValueError(msg) - else: # pragma: no cover - try: - # implementation is UNKNOWN, Narwhals extension using this feature should - # implement `read_parquet` function in the top-level namespace. - native_frame = native_namespace.read_parquet(source=source, **kwargs) - except AttributeError as e: - msg = "Unknown namespace is expected to implement `read_parquet` function." - raise AttributeError(msg) from e + else: + native_frame = _plugin_hook(native_namespace, "read_parquet")( + normalize_path(source), **kwargs + ) return from_native(native_frame, eager_only=True) @@ -942,7 +887,11 @@ def scan_parquet( └──────────────────┘ """ implementation = Implementation.from_backend(backend) - native_namespace = implementation.to_native_namespace() + native_namespace = ( + _backend_namespace(backend) + if implementation is Implementation.UNKNOWN + else implementation.to_native_namespace() + ) native_frame: NativeDataFrame | NativeLazyFrame source = normalize_path(source) if implementation is Implementation.POLARS: @@ -974,14 +923,8 @@ def scan_parquet( else pq_reader.options(**kwargs).load(source) ) - else: # pragma: no cover - try: - # implementation is UNKNOWN, Narwhals extension using this feature should - # implement `scan_parquet` function in the top-level namespace. - native_frame = native_namespace.scan_parquet(source=source, **kwargs) - except AttributeError as e: - msg = "Unknown namespace is expected to implement `scan_parquet` function." - raise AttributeError(msg) from e + else: + native_frame = _plugin_hook(native_namespace, "scan_parquet")(source, **kwargs) return from_native(native_frame).lazy() diff --git a/src/narwhals/plugins.py b/src/narwhals/plugins.py index e550bebca0..79a17f5c3a 100644 --- a/src/narwhals/plugins.py +++ b/src/narwhals/plugins.py @@ -2,6 +2,7 @@ import sys from functools import cache +from types import ModuleType from typing import TYPE_CHECKING, Any, Protocol from narwhals._compliant import CompliantNamespace @@ -20,6 +21,7 @@ CompliantLazyFrameAny, CompliantSeriesAny, ) + from narwhals._typing import Backend, IntoBackend from narwhals.utils import Version @@ -48,6 +50,55 @@ def _discover_entrypoints() -> EntryPoints: return eps(group=group) +def _plugin_names() -> tuple[str, ...]: + """Entry point names of all installed plugins.""" + return tuple(entry_point.name for entry_point in _discover_entrypoints()) + + +def _find_plugin(backend_name: str, /) -> ModuleType | None: + """Return the namespace of the first installed plugin matching `backend_name`. + + `backend_name` is matched against both the entry point name and its module, + e.g. both `"my-plugin"` and `"my_plugin"` for a plugin registered as: + + [project.entry-points.'narwhals.plugins'] + my-plugin = 'my_plugin' + """ + for entry_point in _discover_entrypoints(): + if backend_name in {entry_point.name, entry_point.module}: + namespace: ModuleType = entry_point.load() + return namespace + return None + + +def _backend_namespace(backend: IntoBackend[Backend], /) -> ModuleType: + """Resolve a backend which is not a Narwhals `Implementation` to a plugin namespace. + + The namespace is expected to implement the `Plugin` protocol and/or the extension + hooks required by the calling function (e.g. `scan_csv`, ...) at its top level. + """ + if isinstance(backend, ModuleType): + return backend + if isinstance(backend, str) and (plugin := _find_plugin(backend)) is not None: + return plugin + installed = ", ".join(_plugin_names()) or "" + msg = ( + f"Unsupported backend: {backend!r}.\n\n" + "Expected one of Narwhals' built-in backends (e.g. 'pandas', 'polars', " + "'pyarrow'), a native namespace module, or the name of an installed " + f"Narwhals plugin (installed plugins: {installed})." + ) + raise ValueError(msg) + + +def _plugin_hook(native_namespace: ModuleType, name: str, /) -> Any: + """Fetch extension hook `name` from a plugin namespace, raising if missing.""" + if (hook := getattr(native_namespace, name, None)) is None: + msg = f"Unknown namespace is expected to implement `{name}` function." + raise AttributeError(msg) + return hook + + class PluginNamespace(CompliantNamespace[FrameT, Any], Protocol[FrameT, FromNativeR_co]): def from_native(self, data: Any, /) -> FromNativeR_co: ... diff --git a/src/narwhals/series.py b/src/narwhals/series.py index b4c03ecf90..be5fec05f3 100644 --- a/src/narwhals/series.py +++ b/src/narwhals/series.py @@ -2,7 +2,6 @@ import math from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence -from functools import partial from typing import TYPE_CHECKING, Any, ClassVar, Generic, Literal, cast, overload from narwhals._expression_parsing import ExprKind, ExprNode @@ -13,10 +12,11 @@ _Implementation, _resolve_sample_size, _validate_rolling_arguments, + eager_namespace, + eager_namespace_from_compliant, ensure_type, generate_repr, is_compliant_series, - is_eager_allowed, is_index_selector, qualified_type_name, supports_arrow_c_stream, @@ -165,19 +165,16 @@ def from_numpy( raise ValueError(msg) if dtype: _validate_into_dtype(dtype) - implementation = Implementation.from_backend(backend) - if is_eager_allowed(implementation): - ns = cls._version.namespace.from_backend(implementation).compliant - compliant = ns.from_numpy(values).alias(name) - if dtype: - return cls(compliant.cast(dtype), level="full") - return cls(compliant, level="full") - msg = ( # pragma: no cover - f"{implementation} support in Narwhals is lazy-only, but `Series.from_numpy` is an eager-only function.\n\n" - "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" - f" nw.Series.from_numpy(arr, backend='pyarrow').to_frame().lazy('{implementation}')" + ns = eager_namespace( + backend, + version=cls._version, + function_name="Series.from_numpy", + hint_example="nw.Series.from_numpy(arr, backend='pyarrow').to_frame()", ) - raise ValueError(msg) # pragma: no cover + compliant = ns._series.from_numpy(values, context=ns).alias(name) + if dtype: + return cls(compliant.cast(dtype), level="full") + return cls(compliant, level="full") @classmethod def from_iterable( @@ -227,19 +224,14 @@ def from_iterable( if not isinstance(values, Iterable): msg = f"Expected values to be an iterable, got: {qualified_type_name(values)!r}." raise TypeError(msg) - implementation = Implementation.from_backend(backend) - if is_eager_allowed(implementation): - ns = cls._version.namespace.from_backend(implementation).compliant - compliant = ns._series.from_iterable( - values, context=ns, name=name, dtype=dtype - ) - return cls(compliant, level="full") - msg = ( - f"{implementation} support in Narwhals is lazy-only, but `Series.from_iterable` is an eager-only function.\n\n" - "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" - f" nw.Series.from_iterable('a', [1,2,3], backend='pyarrow').to_frame().lazy('{implementation}')" + ns = eager_namespace( + backend, + version=cls._version, + function_name="Series.from_iterable", + hint_example="nw.Series.from_iterable('a', [1,2,3], backend='pyarrow').to_frame()", ) - raise ValueError(msg) + compliant = ns._series.from_iterable(values, context=ns, name=name, dtype=dtype) + return cls(compliant, level="full") implementation: _Implementation = _Implementation() """Return [`narwhals.Implementation`][] of native Series. @@ -433,9 +425,18 @@ def scatter( a: [[999,888,3]] b: [[4,5,6]] """ - into_series = partial( - type(self).from_iterable, name="", backend=self.implementation - ) + impl = self.implementation + + def into_series(values: Any, dtype: IntoDType | None = None) -> Series[Any]: + if impl is Implementation.UNKNOWN: # type: ignore[comparison-overlap] + ns = eager_namespace_from_compliant( + self._compliant_series, function_name="Series.scatter" + ) + compliant = ns._series.from_iterable( + values, context=ns, name="", dtype=dtype + ) + return self._with_compliant(compliant) + return type(self).from_iterable("", values, dtype, backend=impl) if not isinstance(indices, Series): if not isinstance(indices, Iterable): diff --git a/src/narwhals/testing/asserts/series.py b/src/narwhals/testing/asserts/series.py index 5bdcbb2ea7..80ad2b673f 100644 --- a/src/narwhals/testing/asserts/series.py +++ b/src/narwhals/testing/asserts/series.py @@ -5,20 +5,28 @@ from narwhals._utils import qualified_type_name from narwhals.dependencies import is_narwhals_series -from narwhals.dtypes import Array, Boolean, Categorical, List, String, Struct -from narwhals.functions import new_series +from narwhals.dtypes import Array, Categorical, List, String, Struct from narwhals.testing.asserts.utils import raise_series_assertion_error if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Iterable from typing import TypeAlias from narwhals.series import Series - from narwhals.typing import IntoSeriesT, SeriesT + from narwhals.typing import IntoDType, IntoSeriesT, SeriesT CheckFn: TypeAlias = Callable[[Series[Any], Series[Any]], None] +def _series_like( + reference: SeriesT, values: Iterable[Any], dtype: IntoDType | None +) -> SeriesT: + """Build a Series from `values` using `reference`'s own backend.""" + source = reference._compliant_series + compliant = type(source).from_iterable(values, name="", context=source, dtype=dtype) + return reference._with_compliant(compliant) + + def assert_series_equal( left: Series[IntoSeriesT], right: Series[IntoSeriesT], @@ -161,7 +169,6 @@ def _check_exact_values( categorical_as_str: bool, ) -> None: """Check exact value equality for various data types.""" - left_impl = left.implementation left_dtype, right_dtype = left.dtype, right.dtype is_not_equal_mask: Series[Any] @@ -182,9 +189,9 @@ def _check_exact_values( abs_tol=abs_tol, categorical_as_str=categorical_as_str, ) + # `_check_list_like` raises on the first mismatch; reaching here means equal. _check_list_like(left, right, left_dtype, right_dtype, check_fn=check_fn) - # If `_check_list_like` didn't raise, then every nested element is equal - is_not_equal_mask = new_series("", [False], dtype=Boolean(), backend=left_impl) + return elif isinstance(left_dtype, Struct) and isinstance(right_dtype, Struct): check_fn = partial( assert_series_equal, @@ -196,18 +203,15 @@ def _check_exact_values( abs_tol=abs_tol, categorical_as_str=categorical_as_str, ) + # `_check_struct` raises on the first mismatch; reaching here means equal. _check_struct(left, right, left_dtype, right_dtype, check_fn=check_fn) - # If `_check_struct` didn't raise, then every nested element is equal - is_not_equal_mask = new_series("", [False], dtype=Boolean(), backend=left_impl) + return elif isinstance(left_dtype, Categorical) and isinstance(right_dtype, Categorical): - # If `_check_categorical` didn't raise, then the categories sources/encodings are - # the same, and we can use equality - _not_equal = _check_categorical( - left, right, categorical_as_str=categorical_as_str - ) - is_not_equal_mask = new_series( - "", [_not_equal], dtype=Boolean(), backend=left_impl - ) + # A raise from `_check_categorical` means the categories sources/encodings + # differ; otherwise it returns whether any element differs. + if _check_categorical(left, right, categorical_as_str=categorical_as_str): + raise_series_assertion_error("exact value mismatch", left, right) + return else: is_not_equal_mask = left != right @@ -241,12 +245,11 @@ def _check_list_like( # Check row by row after transforming each array/list into a new series. # Notice that order within the array/list must be the same, regardless of # `check_order` value at the top level. - impl = left_vals.implementation try: - for left_val, right_val in zip(left_vals, right_vals, strict=True): + for lv, rv in zip(left_vals, right_vals, strict=True): check_fn( - new_series("", values=left_val, dtype=left_dtype.inner, backend=impl), - new_series("", values=right_val, dtype=right_dtype.inner, backend=impl), + _series_like(reference=left_vals, values=lv, dtype=left_dtype.inner), + _series_like(reference=right_vals, values=rv, dtype=right_dtype.inner), ) except AssertionError: raise_series_assertion_error("nested value mismatch", left_vals, right_vals) diff --git a/tests/plugins_test.py b/tests/plugins_test.py index eb304910f2..e500181cda 100644 --- a/tests/plugins_test.py +++ b/tests/plugins_test.py @@ -1,27 +1,67 @@ from __future__ import annotations -from typing import TYPE_CHECKING +import types +from typing import TYPE_CHECKING, Any import pytest import narwhals as nw +from tests.utils import PYARROW_VERSION + +plugin_module = pytest.importorskip("test_plugin") if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + from types import ModuleType + + import numpy as np + import pyarrow as pa + from narwhals.plugins import Plugin +BACKEND: Any = "test-plugin" +DATA: dict[str, Any] = {"a": [1, 1, 2], "b": [4, 5, 6]} +ROWS = [{"a": 1, "b": 4}, {"a": 1, "b": 5}, {"a": 2, "b": 6}] + + +def _np_array(data: Any) -> np.ndarray: + pytest.importorskip("numpy") + import numpy as np + + return np.asarray(data) # type: ignore[no-any-return] + + +def _arrow_table() -> pa.Table: + pytest.importorskip("pyarrow") + import pyarrow as pa + + return pa.table(DATA) + + +@pytest.fixture +def csv_path(tmp_path: Path) -> str: + path = tmp_path / "file.csv" + path.write_text("a,b\n1,4\n1,5\n2,6\n", encoding="utf-8") + return str(path) -def test_plugin() -> None: - pytest.importorskip("test_plugin") - df_native = {"a": [1, 1, 2], "b": [4, 5, 6]} - lf = nw.from_native(df_native) # type: ignore[call-overload] + +@pytest.fixture +def parquet_path(tmp_path: Path) -> str: + pq = pytest.importorskip("pyarrow.parquet") + path = str(tmp_path / "file.parquet") + pq.write_table(_arrow_table(), path) + return path + + +def test_plugin_is_lazy() -> None: + lf = nw.from_native(DATA) # type: ignore[call-overload] assert isinstance(lf, nw.LazyFrame) assert lf.columns == ["a", "b"] def test_not_implemented() -> None: - pytest.importorskip("test_plugin") - df_native = {"a": [1, 1, 2], "b": [4, 5, 6]} - lf = nw.from_native(df_native) # type: ignore[call-overload] + lf = nw.from_native(DATA) # type: ignore[call-overload] with pytest.raises( NotImplementedError, match="is not implemented for: 'DictLazyFrame'" ): @@ -29,7 +69,174 @@ def test_not_implemented() -> None: def test_typing() -> None: - pytest.importorskip("test_plugin") import test_plugin _plugin: Plugin = test_plugin + + +@pytest.mark.parametrize( + "backend", + ["test-plugin", "test_plugin", plugin_module], + ids=["entry-point-name", "module-name", "module"], +) +@pytest.mark.parametrize( + ("scan_function", "path_fixture"), + [(nw.scan_csv, "csv_path"), (nw.scan_parquet, "parquet_path")], + ids=["scan_csv", "scan_parquet"], +) +def test_scan_plugin( + request: pytest.FixtureRequest, + scan_function: Callable[..., nw.LazyFrame[Any]], + path_fixture: str, + backend: str | ModuleType, +) -> None: + """`backend` resolves via the entry point name, its module name, or the module itself.""" + lf = scan_function(request.getfixturevalue(path_fixture), backend=backend) + assert isinstance(lf, nw.LazyFrame) + assert lf.columns == ["a", "b"] + + +@pytest.mark.parametrize( + ("read_function", "path_fixture"), + [(nw.read_csv, "csv_path"), (nw.read_parquet, "parquet_path")], + ids=["read_csv", "read_parquet"], +) +def test_read_plugin_lazy_only( + request: pytest.FixtureRequest, + read_function: Callable[..., nw.DataFrame[Any]], + path_fixture: str, +) -> None: + """`test_plugin.from_native` wraps dicts lazily, so eager reads raise.""" + with pytest.raises(TypeError, match="Cannot only use `eager_only`"): + read_function(request.getfixturevalue(path_fixture), backend=BACKEND) + + +@pytest.mark.parametrize( + "make_dataframe", + [ + lambda: nw.from_dict(DATA, backend=BACKEND), + lambda: nw.from_dicts(ROWS, backend=BACKEND), + lambda: nw.from_numpy( + _np_array([[1, 4], [1, 5], [2, 6]]), schema=["a", "b"], backend=BACKEND + ), + pytest.param( + lambda: nw.from_arrow(_arrow_table(), backend=BACKEND), + marks=pytest.mark.skipif(PYARROW_VERSION < (14,), reason="too old"), + ), + lambda: nw.DataFrame.from_dict(DATA, backend=BACKEND), + lambda: nw.DataFrame.from_dicts(ROWS, backend=BACKEND), + lambda: nw.DataFrame.from_numpy( + _np_array([[1, 4], [1, 5], [2, 6]]), schema=["a", "b"], backend=BACKEND + ), + pytest.param( + lambda: nw.DataFrame.from_arrow(_arrow_table(), backend=BACKEND), + marks=pytest.mark.skipif(PYARROW_VERSION < (14,), reason="too old"), + ), + ], +) +def test_eager_dataframe_constructors_plugin( + make_dataframe: Callable[[], nw.DataFrame[Any]], +) -> None: + """Eager constructors dispatch to the plugin's `EagerNamespace`-compliant namespace.""" + df = make_dataframe() + assert isinstance(df, nw.DataFrame) + assert df.to_native() == DATA + + +@pytest.mark.parametrize( + "make_series", + [ + lambda: nw.new_series("a", [1, 2, 3], backend=BACKEND), + lambda: nw.Series.from_iterable("a", [1, 2, 3], backend=BACKEND), + lambda: nw.Series.from_numpy("a", _np_array([1, 2, 3]), backend=BACKEND), + ], +) +def test_eager_series_constructors_plugin( + make_series: Callable[[], nw.Series[Any]], +) -> None: + """Eager constructors dispatch to the plugin's `EagerNamespace`-compliant namespace.""" + s = make_series() + assert isinstance(s, nw.Series) + assert s.name == "a" + assert s.to_native() == [1, 2, 3] + + +def test_series_scatter_plugin() -> None: + """`scatter` constructs indices/values via the plugin's own namespace.""" + s = nw.Series.from_iterable("a", [1, 2, 3], backend=BACKEND) + assert s.scatter([0, 2], [99, 77]).to_native() == [99, 2, 77] + assert s.scatter(1, 50).to_native() == [1, 50, 3] + # Original Series is unchanged, and empty indices are a no-op. + assert s.to_native() == [1, 2, 3] + assert s.scatter([], []).to_native() == [1, 2, 3] + + +def test_dataframe_filter_mask_plugin() -> None: + """`filter(list[bool])` builds the mask series via the plugin's own namespace.""" + df = nw.from_dict(DATA, backend=BACKEND) + with pytest.raises(NotImplementedError, match="'all_horizontal' is not implemented"): + df.filter([True, False, True]) + + +@pytest.mark.parametrize( + ("hook", "call"), + [ + ("scan_csv", lambda backend: nw.scan_csv("x.csv", backend=backend)), + ("read_csv", lambda backend: nw.read_csv("x.csv", backend=backend)), + ("scan_parquet", lambda backend: nw.scan_parquet("x.parquet", backend=backend)), + ("read_parquet", lambda backend: nw.read_parquet("x.parquet", backend=backend)), + ], +) +def test_plugin_missing_io_hook( + hook: str, call: Callable[[types.ModuleType], Any] +) -> None: + """A module without the required IO hook raises an informative AttributeError.""" + empty_namespace = types.ModuleType("empty_plugin") + with pytest.raises(AttributeError, match=f"expected to implement `{hook}` function"): + call(empty_namespace) + + +def test_plugin_missing_narwhals_namespace() -> None: + """Eager functions require the plugin to implement `__narwhals_namespace__`.""" + empty_namespace = types.ModuleType("empty_plugin") + with pytest.raises( + AttributeError, match="expected to implement `__narwhals_namespace__`" + ): + nw.from_dict(DATA, backend=empty_namespace) + + +def _not_implemented_namespace() -> Any: + from narwhals._utils import not_implemented + + class LazyOnlyNamespace: + _series = not_implemented() + _dataframe = not_implemented() + + return LazyOnlyNamespace() + + +@pytest.mark.parametrize("make_namespace", [object, _not_implemented_namespace]) +@pytest.mark.parametrize( + "call", + [ + lambda backend: nw.from_dict(DATA, backend=backend), + lambda backend: nw.from_dicts(ROWS, backend=backend), + lambda backend: nw.new_series("a", [1], backend=backend), + lambda backend: nw.Series.from_iterable("a", [1], backend=backend), + lambda backend: nw.DataFrame.from_dict(DATA, backend=backend), + ], +) +def test_plugin_not_eager_capable( + call: Callable[[types.ModuleType], Any], make_namespace: Callable[[], Any] +) -> None: + """Eager functions require an `EagerNamespace`-compliant plugin namespace.""" + lazy_plugin = types.ModuleType("lazy_plugin") + lazy_plugin.__narwhals_namespace__ = lambda version: make_namespace() # type: ignore[attr-defined] # noqa: ARG005 + with pytest.raises(ValueError, match="does not provide eager support"): + call(lazy_plugin) + + +def test_unknown_backend_raises() -> None: + """A string matching neither a built-in backend nor an installed plugin.""" + with pytest.raises(ValueError, match="Unsupported backend: 'not-a-backend'"): + nw.scan_csv("x.csv", backend="not-a-backend") # type: ignore[arg-type] diff --git a/tests/testing/assert_series_equal_test.py b/tests/testing/assert_series_equal_test.py index 778a314c80..91e6888324 100644 --- a/tests/testing/assert_series_equal_test.py +++ b/tests/testing/assert_series_equal_test.py @@ -380,23 +380,17 @@ def test_categorical_as_str( categorical_as_str: bool, context: AbstractContextManager[Any], ) -> None: - if ( - "polars" in str(constructor_eager) - and POLARS_VERSION >= (1, 32) - and not categorical_as_str - ): + name = str(constructor_eager) + if "polars" in name and POLARS_VERSION >= (1, 32) and not categorical_as_str: # https://github.com/pola-rs/polars/pull/23016 removed StringCache, it still # exists but it does nothing in python. request.applymarker(pytest.mark.xfail) - if "pyarrow_table" in str(constructor_eager) and not categorical_as_str: + if "pyarrow_table" in name and not categorical_as_str: # pyarrow dictionary dtype compares values, not the encoding. request.applymarker(pytest.mark.xfail) - if "pyarrow_table" in str(constructor_eager) and PYARROW_VERSION < ( - 15, - 0, - ): # pragma: no cover + if "pyarrow_table" in name and PYARROW_VERSION < (15, 0): # pragma: no cover reason = ( "pyarrow.lib.ArrowNotImplementedError: Unsupported cast from string to " "dictionary using function cast_dictionary" @@ -415,3 +409,21 @@ def test_categorical_as_str( assert_series_equal( left, right, check_names=False, categorical_as_str=categorical_as_str ) + + +def test_categorical_as_str_value_mismatch(constructor_eager: ConstructorEager) -> None: + name = str(constructor_eager) + if "pyarrow_table" in name and PYARROW_VERSION < (15, 0): # pragma: no cover + reason = ( + "pyarrow.lib.ArrowNotImplementedError: Unsupported cast from string to " + "dictionary using function cast_dictionary" + ) + pytest.skip(reason=reason) + + data = {"a": ["beluga", "orca", "orca", "beluga"]} + frame = nw.from_native(constructor_eager(data), eager_only=True) + s = frame["a"].cast(nw.Categorical()) + left, right = s[:2], s[2:] + + with _assertion_error("exact value mismatch"): + assert_series_equal(left, right)