feat(assertions): add expect.soft() for collecting multiple failures (#3065)

Author: Skn0tt

playwright/_impl/_assertions.py

@@ -13,7 +13,8 @@
 # limitations under the License.
 
 import collections.abc
-from typing import Any, List, Literal, Optional, Pattern, Sequence, Union
+from contextlib import contextmanager
+from typing import Any, Iterator, List, Literal, Optional, Pattern, Sequence, Union
 from urllib.parse import urljoin
 
 from playwright._impl._api_structures import (
@@ -31,6 +32,33 @@
 from playwright._impl._page import Page
 from playwright._impl._str_utils import escape_regex_flags
 
+_soft_errors: Optional[List[AssertionError]] = None
+
+
+@contextmanager
+def _soft_scope() -> Iterator[List[AssertionError]]:
+    global _soft_errors
+    assert _soft_errors is None, "nested soft assertion scopes are not supported"
+    _soft_errors = []
+    try:
+        yield _soft_errors
+    finally:
+        _soft_errors = None
+
+
+def _record_soft_or_raise(error: AssertionError, is_soft: bool) -> None:
+    __tracebackhide__ = True
+    if is_soft:
+        if _soft_errors is None:
+            raise RuntimeError(
+                "expect.soft(...) requires pytest-playwright>=0.7.3 "
+                "(or pytest-playwright-asyncio>=0.7.3). Upgrade the plugin, "
+                "or use a regular expect(...) assertion."
+            )
+        _soft_errors.append(error)
+        return
+    raise error
+
 
 class AssertionsBase:
     def __init__(
@@ -39,13 +67,15 @@ def __init__(
         timeout: float = None,
         is_not: bool = False,
         message: Optional[str] = None,
+        is_soft: bool = False,
     ) -> None:
         self._actual_locator = locator
         self._loop = locator._loop
         self._dispatcher_fiber = locator._dispatcher_fiber
         self._timeout = timeout
         self._is_not = is_not
         self._custom_message = message
+        self._is_soft = is_soft
 
     async def _call_expect(
         self, expression: str, expect_options: FrameExpectOptions, title: Optional[str]
@@ -90,8 +120,11 @@ async def _expect_impl(
                 )
             error_message = result.get("errorMessage")
             error_message = f"\n{error_message}" if error_message else ""
-            raise AssertionError(
-                f"{out_message}\nActual value: {actual}{error_message} {format_call_log(result.get('log'))}"
+            _record_soft_or_raise(
+                AssertionError(
+                    f"{out_message}\nActual value: {actual}{error_message} {format_call_log(result.get('log'))}"
+                ),
+                self._is_soft,
             )
 
 
@@ -102,8 +135,9 @@ def __init__(
         timeout: float = None,
         is_not: bool = False,
         message: Optional[str] = None,
+        is_soft: bool = False,
     ) -> None:
-        super().__init__(page.locator(":root"), timeout, is_not, message)
+        super().__init__(page.locator(":root"), timeout, is_not, message, is_soft)
         self._actual_page = page
 
     async def _call_expect(
@@ -117,7 +151,11 @@ async def _call_expect(
     @property
     def _not(self) -> "PageAssertions":
         return PageAssertions(
-            self._actual_page, self._timeout, not self._is_not, self._custom_message
+            self._actual_page,
+            self._timeout,
+            not self._is_not,
+            self._custom_message,
+            self._is_soft,
         )
 
     async def to_have_title(
@@ -195,8 +233,9 @@ def __init__(
         timeout: float = None,
         is_not: bool = False,
         message: Optional[str] = None,
+        is_soft: bool = False,
     ) -> None:
-        super().__init__(locator, timeout, is_not, message)
+        super().__init__(locator, timeout, is_not, message, is_soft)
         self._actual_locator = locator
 
     async def _call_expect(
@@ -208,7 +247,11 @@ async def _call_expect(
     @property
     def _not(self) -> "LocatorAssertions":
         return LocatorAssertions(
-            self._actual_locator, self._timeout, not self._is_not, self._custom_message
+            self._actual_locator,
+            self._timeout,
+            not self._is_not,
+            self._custom_message,
+            self._is_soft,
         )
 
     async def to_contain_text(
@@ -974,18 +1017,24 @@ def __init__(
         timeout: float = None,
         is_not: bool = False,
         message: Optional[str] = None,
+        is_soft: bool = False,
     ) -> None:
         self._loop = response._loop
         self._dispatcher_fiber = response._dispatcher_fiber
         self._timeout = timeout
         self._is_not = is_not
         self._actual = response
         self._custom_message = message
+        self._is_soft = is_soft
 
     @property
     def _not(self) -> "APIResponseAssertions":
         return APIResponseAssertions(
-            self._actual, self._timeout, not self._is_not, self._custom_message
+            self._actual,
+            self._timeout,
+            not self._is_not,
+            self._custom_message,
+            self._is_soft,
         )
 
     async def to_be_ok(
@@ -1006,7 +1055,7 @@ async def to_be_ok(
         if text is not None:
             out_message += f"\n Response Text:\n{text[:1000]}"
 
-        raise AssertionError(out_message)
+        _record_soft_or_raise(AssertionError(out_message), self._is_soft)
 
     async def not_to_be_ok(self) -> None:
         __tracebackhide__ = True

playwright/async_api/__init__.py

@@ -131,19 +131,68 @@ def __call__(
 
     def __call__(
         self, actual: Union[Page, Locator, APIResponse], message: Optional[str] = None
+    ) -> Union[PageAssertions, LocatorAssertions, APIResponseAssertions]:
+        return self._dispatch(actual, message, is_soft=False)
+
+    @overload
+    def soft(self, actual: Page, message: Optional[str] = None) -> PageAssertions: ...
+
+    @overload
+    def soft(
+        self, actual: Locator, message: Optional[str] = None
+    ) -> LocatorAssertions: ...
+
+    @overload
+    def soft(
+        self, actual: APIResponse, message: Optional[str] = None
+    ) -> APIResponseAssertions: ...
+
+    def soft(
+        self, actual: Union[Page, Locator, APIResponse], message: Optional[str] = None
+    ) -> Union[PageAssertions, LocatorAssertions, APIResponseAssertions]:
+        """
+        Creates a [soft assertion](https://playwright.dev/python/docs/test-assertions#soft-assertions).
+        Failing soft assertions do not abort test execution, but mark the test
+        as failed. Multiple failures from the same test are surfaced together
+        at the end of the test.
+
+        Requires the [pytest-playwright](https://pypi.org/project/pytest-playwright/)
+        plugin to establish the per-test scope that collects soft assertion
+        failures.
+        """
+        return self._dispatch(actual, message, is_soft=True)
+
+    def _dispatch(
+        self,
+        actual: Union[Page, Locator, APIResponse],
+        message: Optional[str],
+        is_soft: bool,
     ) -> Union[PageAssertions, LocatorAssertions, APIResponseAssertions]:
         if isinstance(actual, Page):
             return PageAssertions(
-                PageAssertionsImpl(actual._impl_obj, self._timeout, message=message)
+                PageAssertionsImpl(
+                    actual._impl_obj,
+                    self._timeout,
+                    message=message,
+                    is_soft=is_soft,
+                )
             )
         elif isinstance(actual, Locator):
             return LocatorAssertions(
-                LocatorAssertionsImpl(actual._impl_obj, self._timeout, message=message)
+                LocatorAssertionsImpl(
+                    actual._impl_obj,
+                    self._timeout,
+                    message=message,
+                    is_soft=is_soft,
+                )
             )
         elif isinstance(actual, APIResponse):
             return APIResponseAssertions(
                 APIResponseAssertionsImpl(
-                    actual._impl_obj, self._timeout, message=message
+                    actual._impl_obj,
+                    self._timeout,
+                    message=message,
+                    is_soft=is_soft,
                 )
             )
         raise ValueError(f"Unsupported type: {type(actual)}")

playwright/sync_api/__init__.py

@@ -131,19 +131,68 @@ def __call__(
 
     def __call__(
         self, actual: Union[Page, Locator, APIResponse], message: Optional[str] = None
+    ) -> Union[PageAssertions, LocatorAssertions, APIResponseAssertions]:
+        return self._dispatch(actual, message, is_soft=False)
+
+    @overload
+    def soft(self, actual: Page, message: Optional[str] = None) -> PageAssertions: ...
+
+    @overload
+    def soft(
+        self, actual: Locator, message: Optional[str] = None
+    ) -> LocatorAssertions: ...
+
+    @overload
+    def soft(
+        self, actual: APIResponse, message: Optional[str] = None
+    ) -> APIResponseAssertions: ...
+
+    def soft(
+        self, actual: Union[Page, Locator, APIResponse], message: Optional[str] = None
+    ) -> Union[PageAssertions, LocatorAssertions, APIResponseAssertions]:
+        """
+        Creates a [soft assertion](https://playwright.dev/python/docs/test-assertions#soft-assertions).
+        Failing soft assertions do not abort test execution, but mark the test
+        as failed. Multiple failures from the same test are surfaced together
+        at the end of the test.
+
+        Requires the [pytest-playwright](https://pypi.org/project/pytest-playwright/)
+        plugin to establish the per-test scope that collects soft assertion
+        failures.
+        """
+        return self._dispatch(actual, message, is_soft=True)
+
+    def _dispatch(
+        self,
+        actual: Union[Page, Locator, APIResponse],
+        message: Optional[str],
+        is_soft: bool,
     ) -> Union[PageAssertions, LocatorAssertions, APIResponseAssertions]:
         if isinstance(actual, Page):
             return PageAssertions(
-                PageAssertionsImpl(actual._impl_obj, self._timeout, message=message)
+                PageAssertionsImpl(
+                    actual._impl_obj,
+                    self._timeout,
+                    message=message,
+                    is_soft=is_soft,
+                )
             )
         elif isinstance(actual, Locator):
             return LocatorAssertions(
-                LocatorAssertionsImpl(actual._impl_obj, self._timeout, message=message)
+                LocatorAssertionsImpl(
+                    actual._impl_obj,
+                    self._timeout,
+                    message=message,
+                    is_soft=is_soft,
+                )
             )
         elif isinstance(actual, APIResponse):
             return APIResponseAssertions(
                 APIResponseAssertionsImpl(
-                    actual._impl_obj, self._timeout, message=message
+                    actual._impl_obj,
+                    self._timeout,
+                    message=message,
+                    is_soft=is_soft,
                 )
             )
         raise ValueError(f"Unsupported type: {type(actual)}")

tests/async/test_assertions.py

@@ -1132,3 +1132,50 @@ async def test_to_have_role(page: Page) -> None:
     with pytest.raises(Error) as excinfo:
         await expect(page.locator("div")).to_have_role(re.compile(r"button|checkbox"))  # type: ignore
     assert '"role" argument in to_have_role must be a string' in str(excinfo.value)
+
+
+async def test_soft_outside_scope_raises_runtime_error(
+    page: Page, server: Server
+) -> None:
+    await page.set_content("<div>hello</div>")
+    with pytest.raises(RuntimeError, match="pytest-playwright"):
+        await expect.soft(page.locator("div")).to_have_text("nope", timeout=500)
+
+
+async def test_soft_inside_scope_collects_failures(page: Page, server: Server) -> None:
+    from playwright._impl._assertions import _soft_scope
+
+    await page.goto(server.EMPTY_PAGE)
+    await page.set_content("<title>actual</title><div>hello</div>")
+
+    with _soft_scope() as errors:
+        # should collect, not raise
+        await expect.soft(page).to_have_title("expected", timeout=500)
+        await expect.soft(page.locator("div")).to_have_text("goodbye", timeout=500)
+        # passing soft should not affect the collector
+        await expect.soft(page.locator("div")).to_have_text("hello")
+        # nested .not_ should still be soft
+        await expect.soft(page.locator("div")).not_to_have_text("hello", timeout=500)
+
+    assert len(errors) == 3
+    assert all(isinstance(e, AssertionError) for e in errors)
+
+
+async def test_soft_does_not_leak_between_scopes(page: Page, server: Server) -> None:
+    from playwright._impl._assertions import _soft_scope
+
+    await page.goto(server.EMPTY_PAGE)
+    await page.set_content("<title>actual</title>")
+
+    with _soft_scope() as errors_a:
+        await expect.soft(page).to_have_title("nope", timeout=500)
+    assert len(errors_a) == 1
+
+    with _soft_scope() as errors_b:
+        pass
+    assert errors_b == []
+
+    # After scope ends, soft assertions raise RuntimeError again.
+    await page.set_content("<div>hello</div>")
+    with pytest.raises(RuntimeError, match="pytest-playwright"):
+        await expect.soft(page.locator("div")).to_have_text("nope", timeout=500)

tests/sync/test_assertions.py

@@ -1067,3 +1067,25 @@ def test_to_have_role(page: Page) -> None:
     with pytest.raises(Error) as excinfo:
         expect(page.locator("div")).to_have_role(re.compile(r"button|checkbox"))  # type: ignore
     assert '"role" argument in to_have_role must be a string' in str(excinfo.value)
+
+
+def test_soft_outside_scope_raises_runtime_error(page: Page, server: Server) -> None:
+    page.set_content("<div>hello</div>")
+    with pytest.raises(RuntimeError, match="pytest-playwright"):
+        expect.soft(page.locator("div")).to_have_text("nope", timeout=500)
+
+
+def test_soft_inside_scope_collects_failures(page: Page, server: Server) -> None:
+    from playwright._impl._assertions import _soft_scope
+
+    page.goto(server.EMPTY_PAGE)
+    page.set_content("<title>actual</title><div>hello</div>")
+
+    with _soft_scope() as errors:
+        expect.soft(page).to_have_title("expected", timeout=500)
+        expect.soft(page.locator("div")).to_have_text("goodbye", timeout=500)
+        expect.soft(page.locator("div")).to_have_text("hello")
+        expect.soft(page.locator("div")).not_to_have_text("hello", timeout=500)
+
+    assert len(errors) == 3
+    assert all(isinstance(e, AssertionError) for e in errors)