diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb45ab60..f2246934 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,9 @@ jobs: - python-version: "3.14" uv-resolution: "highest" extra: "--extra structlog" + - python-version: "3.14" + uv-resolution: "highest" + extra: "--extra pydantic" - python-version: "3.14" uv-resolution: "highest" extra: "--extra twisted --group twisted-dev" @@ -110,6 +113,7 @@ jobs: - "sybil" - "mock" - "structlog" + - "pydantic" - "twisted" steps: - uses: cjw296/python-action/check-distributions@v3 diff --git a/docs/api.rst b/docs/api.rst index e3a17bef..3f9a9dff 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -296,6 +296,12 @@ testfixtures.polars .. automodule:: testfixtures.polars :members: +testfixtures.pydantic +~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: testfixtures.pydantic + :members: + testfixtures.structlog ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/comparing.rst b/docs/comparing.rst index d1660b1f..302fa72e 100644 --- a/docs/comparing.rst +++ b/docs/comparing.rst @@ -260,50 +260,75 @@ This is handled for standard container types and their subclasses, specifically :class:`tuple`, :class:`dict`, :class:`set`, and :class:`frozenset`. For custom container or wrapper types that implement ``__eq__`` and don't subclass one -of these standard container types, *and* which can contain instance of a type for which you'd like -to ``ignore_eq``, you will find that ``ignore_eq`` for that type alone is not sufficient. +of these standard container types, *and* which can contain instances of a type for which you'd like +to ``ignore_eq``, you will find that ``ignore_eq`` for the inner type alone is not sufficient. -For example, consider this container type: +A real-world example is a :class:`pydantic.BaseModel ` with a +:class:`pandas.DataFrame` attribute. :class:`~pandas.DataFrame` implements ``__eq__`` in a way +that raises :exc:`ValueError` when used as a boolean, and pydantic's ``__eq__`` calls ``==`` on +each field value directly, so if :class:`~pandas.DataFrame` is the only type passed to +``ignore_eq``, pydantic's ``__eq__`` still fires first and the comparison still raises: .. code-block:: python - class OrmObjBox: - def __init__(self, *items: OrmObj): - self.items = list(items) - def __eq__(self, other: "OrmObjBox") -> bool: - return self.items == other.items + import pandas as pd + from pydantic import BaseModel, ConfigDict -If we only pass :class:`!OrmObj` to ``ignore_eq``, the :class:`!OrmObjBox` instances will still -erroneously appear to be equal: + class Report(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) + name: str + data: pd.DataFrame ->>> compare(OrmObjBox(OrmObj(1)), OrmObjBox(OrmObj(2)), ignore_eq=OrmObj) +.. invisible-code-block: python -To successfully ignore the ``__eq__`` of :class:`!OrmObj`, we need to pass both the type and -any custom container or wrapper types to ``ignore_eq``: + from testfixtures.comparing import Registry + registry = Registry.initial().install() ->>> compare(OrmObjBox(OrmObj(1)), OrmObjBox(OrmObj(2)), ignore_eq=[OrmObj, OrmObjBox]) +>>> compare( +... Report(name='sales', data=pd.DataFrame({'x': [1, 2]})), +... expected=Report(name='sales', data=pd.DataFrame({'x': [1, 3]})), +... ignore_eq=pd.DataFrame, +... ) Traceback (most recent call last): ... -AssertionError: OrmObjBox not as expected: - -attributes differ: -'items': [OrmObj: 1] != [OrmObj: 2] - -While comparing .items: sequence not as expected: - -same: -[] +ValueError: The truth value of a DataFrame is ambiguous. Use a.empty, a.bool(), a.item(), a.any() or a.all(). + +.. invisible-code-block: python + + registry.uninstall() + +Both types must be known to :func:`compare`. For broadly applicable types like +:class:`~pydantic.BaseModel` and :class:`~pandas.DataFrame`, the right solution is to +:ref:`register ` a comparer for each. This happens automatically +when both pydantic and pandas are installed, so :func:`compare` handles +:class:`~pydantic.BaseModel` instances containing :class:`~pandas.DataFrame` +attributes without any extra arguments: + +>>> compare( +... Report(name='sales', data=pd.DataFrame({'x': [1, 2]})), +... expected=Report(name='sales', data=pd.DataFrame({'x': [1, 3]})), +... ) +Traceback (most recent call last): +... +AssertionError: Report not as expected: -first: -[OrmObj: 1] +attributes same: +['name'] -second: -[OrmObj: 2] +attributes differ: +'data': x +0 1 +1 3 (expected) != x +0 1 +1 2 (actual) -While comparing .items[0]: OrmObj not as expected: +While comparing .data: DataFrame.iloc[:, 0] (column name="x") are different -attributes differ: -'a': 1 != 2 +DataFrame.iloc[:, 0] (column name="x") values are different (50.0 %) +[index]: [0, 1] +[left]: [1, 3] +[right]: [1, 2] +At positional index 1, first diff: 3 != 2 .. _recursion: diff --git a/docs/conf.py b/docs/conf.py index d3043931..0f521868 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -18,6 +18,7 @@ 'numpy': ('https://numpy.org/doc/stable/', None), 'pandas': ('https://pandas.pydata.org/docs/', None), 'polars': ('https://docs.pola.rs/api/python/stable/', None), + 'pydantic': ('https://docs.pydantic.dev/latest/', None), } # General diff --git a/docs/exceptions.rst b/docs/exceptions.rst index 64ac86ec..85bdf25d 100644 --- a/docs/exceptions.rst +++ b/docs/exceptions.rst @@ -3,6 +3,13 @@ Testing exceptions .. currentmodule:: testfixtures +.. invisible-code-block: python + + try: + import pydantic + except ImportError: + pydantic = None + Testfixtures has tools to help when making assertions about exceptions that should be raised by a piece of code. @@ -174,31 +181,56 @@ Matching the type and ``repr()`` or ``str()`` Sometimes you want to assert both the type of the exception and its :func:`repr` or :class:`str` without constructing the exception instance yourself. The :func:`repr_like` and :func:`str_like` matchers do this and are -typed to stand in for the exception: +typed to stand in for the exception. + +This is particularly useful for exceptions such as +:class:`pydantic.ValidationError `, which has no public +constructor that accepts a plain message: + +.. skip: start if(pydantic is None, reason="No pydantic installed") + +.. code-block:: python + + from pydantic import BaseModel, ValidationError + + class Point(BaseModel): + x: int + y: int >>> from testfixtures import repr_like ->>> with ShouldRaise(repr_like(ValueError, "ValueError('Not good!')")): -... the_thrower() +>>> with ShouldRaise(repr_like(ValidationError, match='validation error for Point')): +... Point(x='not-an-int', y=2) >>> from testfixtures import str_like ->>> with ShouldRaise(str_like(ValueError, 'Not good!')): -... the_thrower() +>>> with ShouldRaise(str_like(ValidationError, match='validation error for Point')): +... Point(x='not-an-int', y=2) -Both can take a ``match`` regular expression instead of an exact string: +Both matchers can also compare the whole rendering exactly, rather than +matching a pattern within it. If the type matches but the rendering does not, +an :class:`AssertionError` explains the difference: ->>> with ShouldRaise(repr_like(ValueError, match=r'good')): -... the_thrower() +.. invisible-code-block: python -If the type matches but the rendering does not, an :class:`AssertionError` -explains the difference: + try: + Point(x='not-an-int', y=2) + except ValidationError as e: + wrong_text = str(e).replace('\nx\n', '\ny\n', 1) ->>> with ShouldRaise(str_like(ValueError, 'All good!')): -... the_thrower() +>>> with ShouldRaise(str_like(ValidationError, wrong_text)): +... Point(x='not-an-int', y=2) Traceback (most recent call last): ... AssertionError: not equal: - (expected) -ValueError('Not good!') (raised) + (expected) +1 validation error for Point +x + Input should be a valid integer, unable to parse string as an integer [type=int_parsing, input_value='not-an-int', input_type=str] + For further information visit https://errors.pydantic.dev/.../v/int_parsing (raised) + +.. skip: end The type must match exactly, so a subclass of the expected exception will not match. See :ref:`comparison-objects` for more about :func:`repr_like` and diff --git a/docs/index.rst b/docs/index.rst index 519adfbd..73984cd8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -22,6 +22,7 @@ The sections below describe the use of the various tools included: pandas.rst numpy.rst structlog.rst + pydantic.rst twisted.rst utilities.rst diff --git a/docs/installation.rst b/docs/installation.rst index 4d75ce27..2c6d5cb0 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -75,6 +75,8 @@ with the version of these packages that your project is using: - ``[structlog]``: Structlog logging framework support. See :doc:`structlog`. +- ``[pydantic]``: Pydantic model helpers. See :doc:`pydantic`. + - ``[yaml]``: YAML format support for :class:`~testfixtures.TempDir`. See :ref:`files`. - ``[toml]``: TOML format writing support for :class:`~testfixtures.TempDir`. See :ref:`files`. diff --git a/docs/pydantic.rst b/docs/pydantic.rst new file mode 100644 index 00000000..e8687d4e --- /dev/null +++ b/docs/pydantic.rst @@ -0,0 +1,188 @@ +Testing with Pydantic +===================== + +.. note:: + + To ensure you are using compatible versions, install with the ``testfixtures[pydantic]`` extra. + +.. invisible-code-block: python + + try: + import pydantic + except ImportError: + pydantic = None + +.. skip: start if(pydantic is None, reason="No pydantic installed") + +When pydantic is installed, a :func:`comparer ` +for :class:`~pydantic.BaseModel` is automatically +:ref:`registered ` with ``ignore_eq=True``. It compares +models field-by-field using their declared attributes, so differences are +shown clearly: + +>>> from pydantic import BaseModel +>>> from testfixtures import compare +>>> class Point(BaseModel): +... x: int +... y: int +>>> compare(Point(x=1, y=2), expected=Point(x=1, y=3)) +Traceback (most recent call last): + ... +AssertionError: Point not as expected: + +attributes same: +['x'] + +attributes differ: +'y': 3 (expected) != 2 (actual) + +The ``ignore_eq=True`` registration is also needed whenever a model contains +attributes whose type has a custom ``__eq__`` that :func:`~testfixtures.compare` +has a registered comparer for, such as :doc:`Polars ` or +:doc:`Pandas ` DataFrames. Without it, pydantic's ``__eq__`` +calls ``==`` on each attribute value before testfixtures can intercept it, +which for many such types raises an error or gives a misleading result. + +For example, consider a model with a :class:`~polars.DataFrame` attribute: + +.. code-block:: python + + import polars as pl + from pydantic import BaseModel, ConfigDict + + class Report(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) + name: str + data: pl.DataFrame + + r1 = Report(name='sales', data=pl.DataFrame({'x': [1, 2], 'y': [3, 4]})) + r2 = Report(name='sales', data=pl.DataFrame({'x': [1, 2], 'y': [3, 5]})) + +Without the ``BaseModel`` registration, pydantic's ``__eq__`` fires first and +raises a :exc:`TypeError`: + +.. invisible-code-block: python + + from testfixtures.comparing import Registry + registry = Registry.initial().install() + +>>> compare(r1, expected=r2) +Traceback (most recent call last): + ... +TypeError: the truth value of a DataFrame is ambiguous + +Hint: to check if a DataFrame contains any values, use `is_empty()`. + +.. invisible-code-block: python + + registry.uninstall() + +With the registration in place, :func:`~testfixtures.compare` hands off +to the :doc:`Polars comparer ` and produces a clear diff: + +>>> compare(r1, expected=r2) +Traceback (most recent call last): + ... +AssertionError: Report not as expected: + +attributes same: +['name'] + +attributes differ: +'data': shape: (2, 2) +┌─────┬─────┐ +│ x ┆ y │ +│ --- ┆ --- │ +│ i64 ┆ i64 │ +╞═════╪═════╡ +│ 1 ┆ 3 │ +│ 2 ┆ 5 │ +└─────┴─────┘ (expected) != shape: (2, 2) +┌─────┬─────┐ +│ x ┆ y │ +│ --- ┆ --- │ +│ i64 ┆ i64 │ +╞═════╪═════╡ +│ 1 ┆ 3 │ +│ 2 ┆ 4 │ +└─────┴─────┘ (actual) + +While comparing .data: DataFrames are different (value mismatch for column "y") +[left]: shape: (2,) +Series: 'y' [i64] +[ + 3 + 5 +] +[right]: shape: (2,) +Series: 'y' [i64] +[ + 3 + 4 +] + +Testing validation errors +-------------------------- + +When a :class:`~pydantic.BaseModel` is given invalid data, pydantic raises +:class:`pydantic.ValidationError `. This type has no public constructor that +takes a plain message, so building an instance to hand to :class:`~testfixtures.ShouldRaise` +for a full comparison is impractical. + +Even setting aside how hard it is to construct one, comparing full instances +would not catch anything useful anyway. Pydantic stores the details of what +went wrong outside the ``args`` and ``__dict__`` that :func:`~testfixtures.compare` inspects +for exceptions, so any two naturally raised :class:`pydantic.ValidationError ` +instances compare equal regardless of what actually failed: + +.. invisible-code-block: python + + from pydantic import ValidationError + + errors = [] + for bad in ({'x': 'not-an-int', 'y': 2}, {'x': 2, 'y': 'also-not-an-int'}): + try: + Point(**bad) + except ValidationError as e: + errors.append(e) + +>>> compare(errors[0], expected=errors[1]) + +The way to check a raised :class:`pydantic.ValidationError ` is therefore to +match its :func:`repr` or :class:`str` rendering with +:func:`~testfixtures.repr_like` or :func:`~testfixtures.str_like`. Use +``match`` rather than ``expected``, since the rendering includes a +pydantic-version-specific URL: + +>>> from testfixtures import ShouldRaise, repr_like +>>> with ShouldRaise(repr_like(ValidationError, match='validation error for Point')): +... Point(x='not-an-int', y=2) + +>>> from testfixtures import str_like +>>> with ShouldRaise(str_like(ValidationError, match='validation error for Point')): +... Point(x='not-an-int', y=2) + +``repr_like`` and ``str_like`` can also compare the whole rendering exactly, +rather than matching a pattern within it. If the rendering doesn't match, an +:class:`AssertionError` explains the difference: + +.. invisible-code-block: python + + try: + Point(x='not-an-int', y=2) + except ValidationError as e: + wrong_text = str(e).replace('\nx\n', '\ny\n', 1) + +>>> with ShouldRaise(repr_like(ValidationError, wrong_text)): +... Point(x='not-an-int', y=2) +Traceback (most recent call last): +... +AssertionError: not equal: + (expected) +1 validation error for Point +x + Input should be a valid integer, unable to parse string as an integer [type=int_parsing, input_value='not-an-int', input_type=str] + For further information visit https://errors.pydantic.dev/.../v/int_parsing (raised) diff --git a/pyproject.toml b/pyproject.toml index 130db691..cfdc7c34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ polars = ["polars>=1.32"] pandas = ["pandas>=2.3.3"] numpy = ["numpy>=2.3.2"] structlog = ["structlog>=24.3.0"] +pydantic = ["pydantic>=2.12.0"] twisted = ["twisted>=22.1"] yaml = ["pyyaml>=6.0.3"] toml = ["tomlkit>=0.10.0"] diff --git a/src/testfixtures/comparing.py b/src/testfixtures/comparing.py index 4e5cffc0..ecec18d1 100644 --- a/src/testfixtures/comparing.py +++ b/src/testfixtures/comparing.py @@ -510,3 +510,12 @@ def compare( else: register(ndarray, compare_ndarray, ignore_eq=True) register(MaskedArray, compare_masked_array, ignore_eq=True) + + +try: + from pydantic import BaseModel as PydanticBaseModel + from .pydantic import compare_basemodel +except ImportError: + pass +else: + register(PydanticBaseModel, compare_basemodel, ignore_eq=True) diff --git a/src/testfixtures/pydantic.py b/src/testfixtures/pydantic.py new file mode 100644 index 00000000..0884b421 --- /dev/null +++ b/src/testfixtures/pydantic.py @@ -0,0 +1,31 @@ +""" +Tools for helping to test applications that use Pydantic. +""" +from typing import TYPE_CHECKING + +import pydantic as pydantic +from pydantic import BaseModel + +from .comparers import _compare_mapping, compare_simple + +if TYPE_CHECKING: + from .comparing import CompareContext + + +def compare_basemodel( + x: BaseModel, + y: BaseModel, + context: 'CompareContext', +) -> str | None: + """ + Returns an informative string describing the differences between the two + supplied Pydantic :class:`~pydantic.BaseModel` instances, based on their + declared fields. + """ + if type(x) is not type(y): + return compare_simple(x, y, context) + x_attrs = x.__dict__.copy() + y_attrs = y.__dict__.copy() + if not context.qualified_equals(x_attrs, y_attrs): + return _compare_mapping(x_attrs, y_attrs, context, x, 'attributes ', '.%s') + return None diff --git a/tests/test_pydantic.py b/tests/test_pydantic.py new file mode 100644 index 00000000..760ee6ca --- /dev/null +++ b/tests/test_pydantic.py @@ -0,0 +1,10 @@ +import pytest + +pytest.importorskip("pydantic") + +import testfixtures.pydantic +from testfixtures import compare + + +def test_importable(): + compare(testfixtures.pydantic.pydantic.__name__, expected="pydantic")