feat: aggregate expect.soft() failures per test

pytest-playwright-asyncio/pytest_playwright_asyncio/pytest_playwright.py

@@ -37,6 +37,16 @@
 )
 
 import pytest
+
+if sys.version_info >= (3, 11):
+    _BaseExceptionGroup = BaseExceptionGroup  # noqa: F821
+else:
+    from exceptiongroup import BaseExceptionGroup as _BaseExceptionGroup
+
+try:
+    from playwright._impl._assertions import _soft_scope
+except ImportError:
+    _soft_scope = None
 from playwright.async_api import (
     Browser,
     BrowserContext,
@@ -89,6 +99,30 @@ def delete_output_dir(pytestconfig: Any) -> None:
                 shutil.rmtree(entry)
 
 
+@pytest.hookimpl(wrapper=True, tryfirst=True)
+def pytest_runtest_call(item: Any) -> Generator[None, Any, None]:
+    if _soft_scope is None:
+        yield
+        return
+    hard_failure: Optional[BaseException] = None
+    with _soft_scope() as errors:
+        try:
+            yield
+        except BaseException as exc:
+            hard_failure = exc
+    if not errors:
+        if hard_failure is not None:
+            raise hard_failure
+        return
+    if hard_failure is not None:
+        raise _BaseExceptionGroup(
+            "Test and soft assertion failures", [hard_failure, *errors]
+        )
+    if len(errors) == 1:
+        raise errors[0]
+    raise _BaseExceptionGroup("Soft assertion failures", errors)
+
+
 def pytest_generate_tests(metafunc: Any) -> None:
     if "browser_name" in metafunc.fixturenames:
         browsers = metafunc.config.option.browser or ["chromium"]

pytest-playwright/pytest_playwright/pytest_playwright.py

@@ -35,6 +35,16 @@
 )
 
 import pytest
+
+if sys.version_info >= (3, 11):
+    _BaseExceptionGroup = BaseExceptionGroup  # noqa: F821
+else:
+    from exceptiongroup import BaseExceptionGroup as _BaseExceptionGroup
+
+try:
+    from playwright._impl._assertions import _soft_scope
+except ImportError:
+    _soft_scope = None
 from playwright.sync_api import (
     Browser,
     BrowserContext,
@@ -86,6 +96,30 @@ def delete_output_dir(pytestconfig: Any) -> None:
                 shutil.rmtree(entry)
 
 
+@pytest.hookimpl(wrapper=True, tryfirst=True)
+def pytest_runtest_call(item: Any) -> Generator[None, Any, None]:
+    if _soft_scope is None:
+        yield
+        return
+    hard_failure: Optional[BaseException] = None
+    with _soft_scope() as errors:
+        try:
+            yield
+        except BaseException as exc:
+            hard_failure = exc
+    if not errors:
+        if hard_failure is not None:
+            raise hard_failure
+        return
+    if hard_failure is not None:
+        raise _BaseExceptionGroup(
+            "Test and soft assertion failures", [hard_failure, *errors]
+        )
+    if len(errors) == 1:
+        raise errors[0]
+    raise _BaseExceptionGroup("Soft assertion failures", errors)
+
+
 def pytest_generate_tests(metafunc: Any) -> None:
     if "browser_name" in metafunc.fixturenames:
         browsers = metafunc.config.option.browser or ["chromium"]

tests/test_asyncio.py

@@ -1091,3 +1091,80 @@ async def test_connect_options(page):
         else:
             os.kill(server_process.pid, signal.SIGINT)
         server_process.wait()
+
+
+def test_soft_assertion_single_failure(testdir: pytest.Testdir) -> None:
+    testdir.makepyfile(
+        """
+        import pytest
+        from playwright.async_api import expect
+
+        @pytest.mark.asyncio
+        async def test_soft(page):
+            await page.set_content("<div>hello</div>")
+            await expect.soft(page.locator("div")).to_have_text("goodbye")
+    """
+    )
+    result = testdir.runpytest("--browser", "chromium")
+    result.assert_outcomes(failed=1)
+    assert any("goodbye" in line for line in result.outlines)
+
+
+def test_soft_assertion_multiple_failures_exception_group(
+    testdir: pytest.Testdir,
+) -> None:
+    testdir.makepyfile(
+        """
+        import pytest
+        from playwright.async_api import expect
+
+        @pytest.mark.asyncio
+        async def test_soft(page):
+            await page.set_content("<div>hello</div>")
+            await expect.soft(page.locator("div")).to_have_text("first")
+            await expect.soft(page.locator("div")).to_have_text("second")
+    """
+    )
+    result = testdir.runpytest("--browser", "chromium")
+    result.assert_outcomes(failed=1)
+    out = "\n".join(result.outlines)
+    assert "first" in out and "second" in out
+
+
+def test_soft_assertion_passes_when_all_match(testdir: pytest.Testdir) -> None:
+    testdir.makepyfile(
+        """
+        import pytest
+        from playwright.async_api import expect
+
+        @pytest.mark.asyncio
+        async def test_soft(page):
+            await page.set_content("<div>hello</div>")
+            await expect.soft(page.locator("div")).to_have_text("hello")
+    """
+    )
+    result = testdir.runpytest("--browser", "chromium")
+    result.assert_outcomes(passed=1)
+
+
+def test_soft_assertion_does_not_shadow_body_failure(
+    testdir: pytest.Testdir,
+) -> None:
+    testdir.makepyfile(
+        """
+        import pytest
+        from playwright.async_api import expect
+
+        @pytest.mark.asyncio
+        async def test_soft(page):
+            await page.set_content("<div>hello</div>")
+            await expect.soft(page.locator("div")).to_have_text("soft-fail")
+            raise RuntimeError("body-fail")
+    """
+    )
+    result = testdir.runpytest("--browser", "chromium")
+    # Body and soft assertion failures are grouped in call phase.
+    result.assert_outcomes(failed=1)
+    out = "\n".join(result.outlines)
+    assert "body-fail" in out
+    assert "soft-fail" in out

tests/test_sync.py

@@ -1077,3 +1077,72 @@ def test_connect_options(page):
         else:
             os.kill(server_process.pid, signal.SIGINT)
         server_process.wait()
+
+
+def test_soft_assertion_single_failure(testdir: pytest.Testdir) -> None:
+    testdir.makepyfile(
+        """
+        from playwright.sync_api import expect
+
+        def test_soft(page):
+            page.set_content("<div>hello</div>")
+            expect.soft(page.locator("div")).to_have_text("goodbye")
+    """
+    )
+    result = testdir.runpytest("--browser", "chromium")
+    result.assert_outcomes(failed=1)
+    assert any("goodbye" in line for line in result.outlines)
+
+
+def test_soft_assertion_multiple_failures_exception_group(
+    testdir: pytest.Testdir,
+) -> None:
+    testdir.makepyfile(
+        """
+        from playwright.sync_api import expect
+
+        def test_soft(page):
+            page.set_content("<div>hello</div>")
+            expect.soft(page.locator("div")).to_have_text("first")
+            expect.soft(page.locator("div")).to_have_text("second")
+    """
+    )
+    result = testdir.runpytest("--browser", "chromium")
+    result.assert_outcomes(failed=1)
+    out = "\n".join(result.outlines)
+    assert "first" in out and "second" in out
+
+
+def test_soft_assertion_passes_when_all_match(testdir: pytest.Testdir) -> None:
+    testdir.makepyfile(
+        """
+        from playwright.sync_api import expect
+
+        def test_soft(page):
+            page.set_content("<div>hello</div>")
+            expect.soft(page.locator("div")).to_have_text("hello")
+    """
+    )
+    result = testdir.runpytest("--browser", "chromium")
+    result.assert_outcomes(passed=1)
+
+
+def test_soft_assertion_does_not_shadow_body_failure(
+    testdir: pytest.Testdir,
+) -> None:
+    testdir.makepyfile(
+        """
+        from playwright.sync_api import expect
+
+        def test_soft(page):
+            page.set_content("<div>hello</div>")
+            expect.soft(page.locator("div")).to_have_text("soft-fail")
+            raise RuntimeError("body-fail")
+    """
+    )
+    result = testdir.runpytest("--browser", "chromium")
+    # Body and soft assertion failures are grouped in call phase.
+    result.assert_outcomes(failed=1)
+    out = "\n".join(result.outlines)
+    assert "body-fail" in out
+    assert "soft-fail" in out