From 0fbbf985ba031d9adf9432edd91c33dfd93100a5 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Mon, 4 May 2026 15:31:34 +0200 Subject: [PATCH 1/3] feat: aggregate expect.soft() failures per test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an autouse function-scoped fixture in both pytest-playwright and pytest-playwright-asyncio that wraps each test in a soft-assertion collection scope. At end of test, collected failures are re-raised: zero → noop, one → re-raise, multiple → BaseExceptionGroup. Requires playwright-python with the soft-assertions hook (microsoft/playwright-python#1272). --- .../pytest_playwright.py | 18 +++++ .../pytest_playwright/pytest_playwright.py | 18 +++++ tests/test_asyncio.py | 54 +++++++++++++++ tests/test_sync.py | 69 +++++++++++++++++++ 4 files changed, 159 insertions(+) diff --git a/pytest-playwright-asyncio/pytest_playwright_asyncio/pytest_playwright.py b/pytest-playwright-asyncio/pytest_playwright_asyncio/pytest_playwright.py index 2917e04..a3cf18f 100644 --- a/pytest-playwright-asyncio/pytest_playwright_asyncio/pytest_playwright.py +++ b/pytest-playwright-asyncio/pytest_playwright_asyncio/pytest_playwright.py @@ -37,6 +37,7 @@ ) import pytest +from playwright._impl._assertions import _soft_scope from playwright.async_api import ( Browser, BrowserContext, @@ -89,6 +90,23 @@ def delete_output_dir(pytestconfig: Any) -> None: shutil.rmtree(entry) +if sys.version_info >= (3, 11): + _BaseExceptionGroup = BaseExceptionGroup # noqa: F821 +else: + from exceptiongroup import BaseExceptionGroup as _BaseExceptionGroup + + +@pytest.fixture(autouse=True) +def _playwright_soft_assertions() -> Generator[None, None, None]: + with _soft_scope() as errors: + yield + if not errors: + return + if len(errors) == 1: + raise errors[0] + raise _BaseExceptionGroup("Soft assertion failures", errors) + + def pytest_generate_tests(metafunc: Any) -> None: if "browser_name" in metafunc.fixturenames: browsers = metafunc.config.option.browser or ["chromium"] diff --git a/pytest-playwright/pytest_playwright/pytest_playwright.py b/pytest-playwright/pytest_playwright/pytest_playwright.py index 058d5d0..d68ae1c 100644 --- a/pytest-playwright/pytest_playwright/pytest_playwright.py +++ b/pytest-playwright/pytest_playwright/pytest_playwright.py @@ -35,6 +35,7 @@ ) import pytest +from playwright._impl._assertions import _soft_scope from playwright.sync_api import ( Browser, BrowserContext, @@ -86,6 +87,23 @@ def delete_output_dir(pytestconfig: Any) -> None: shutil.rmtree(entry) +if sys.version_info >= (3, 11): + _BaseExceptionGroup = BaseExceptionGroup # noqa: F821 +else: + from exceptiongroup import BaseExceptionGroup as _BaseExceptionGroup + + +@pytest.fixture(autouse=True) +def _playwright_soft_assertions() -> Generator[None, None, None]: + with _soft_scope() as errors: + yield + if not errors: + return + if len(errors) == 1: + raise errors[0] + raise _BaseExceptionGroup("Soft assertion failures", errors) + + def pytest_generate_tests(metafunc: Any) -> None: if "browser_name" in metafunc.fixturenames: browsers = metafunc.config.option.browser or ["chromium"] diff --git a/tests/test_asyncio.py b/tests/test_asyncio.py index 539a883..2494936 100644 --- a/tests/test_asyncio.py +++ b/tests/test_asyncio.py @@ -1091,3 +1091,57 @@ async def test_connect_options(page): else: os.kill(server_process.pid, signal.SIGINT) server_process.wait() + + +def test_soft_assertion_single_failure(testdir: pytest.Testdir) -> None: + testdir.makepyfile( + """ + import pytest + from playwright.async_api import expect + + @pytest.mark.asyncio + async def test_soft(page): + await page.set_content("
hello
") + await expect.soft(page.locator("div")).to_have_text("goodbye") + """ + ) + result = testdir.runpytest("--browser", "chromium") + result.assert_outcomes(passed=1, errors=1) + assert any("goodbye" in line for line in result.outlines) + + +def test_soft_assertion_multiple_failures_exception_group( + testdir: pytest.Testdir, +) -> None: + testdir.makepyfile( + """ + import pytest + from playwright.async_api import expect + + @pytest.mark.asyncio + async def test_soft(page): + await page.set_content("
hello
") + await expect.soft(page.locator("div")).to_have_text("first") + await expect.soft(page.locator("div")).to_have_text("second") + """ + ) + result = testdir.runpytest("--browser", "chromium") + result.assert_outcomes(passed=1, errors=1) + out = "\n".join(result.outlines) + assert "first" in out and "second" in out + + +def test_soft_assertion_passes_when_all_match(testdir: pytest.Testdir) -> None: + testdir.makepyfile( + """ + import pytest + from playwright.async_api import expect + + @pytest.mark.asyncio + async def test_soft(page): + await page.set_content("
hello
") + await expect.soft(page.locator("div")).to_have_text("hello") + """ + ) + result = testdir.runpytest("--browser", "chromium") + result.assert_outcomes(passed=1) diff --git a/tests/test_sync.py b/tests/test_sync.py index 374d919..59f6733 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -1077,3 +1077,72 @@ def test_connect_options(page): else: os.kill(server_process.pid, signal.SIGINT) server_process.wait() + + +def test_soft_assertion_single_failure(testdir: pytest.Testdir) -> None: + testdir.makepyfile( + """ + from playwright.sync_api import expect + + def test_soft(page): + page.set_content("
hello
") + expect.soft(page.locator("div")).to_have_text("goodbye") + """ + ) + result = testdir.runpytest("--browser", "chromium") + result.assert_outcomes(passed=1, errors=1) + assert any("goodbye" in line for line in result.outlines) + + +def test_soft_assertion_multiple_failures_exception_group( + testdir: pytest.Testdir, +) -> None: + testdir.makepyfile( + """ + from playwright.sync_api import expect + + def test_soft(page): + page.set_content("
hello
") + expect.soft(page.locator("div")).to_have_text("first") + expect.soft(page.locator("div")).to_have_text("second") + """ + ) + result = testdir.runpytest("--browser", "chromium") + result.assert_outcomes(passed=1, errors=1) + out = "\n".join(result.outlines) + assert "first" in out and "second" in out + + +def test_soft_assertion_passes_when_all_match(testdir: pytest.Testdir) -> None: + testdir.makepyfile( + """ + from playwright.sync_api import expect + + def test_soft(page): + page.set_content("
hello
") + expect.soft(page.locator("div")).to_have_text("hello") + """ + ) + result = testdir.runpytest("--browser", "chromium") + result.assert_outcomes(passed=1) + + +def test_soft_assertion_does_not_shadow_body_failure( + testdir: pytest.Testdir, +) -> None: + testdir.makepyfile( + """ + from playwright.sync_api import expect + + def test_soft(page): + page.set_content("
hello
") + expect.soft(page.locator("div")).to_have_text("soft-fail") + raise RuntimeError("body-fail") + """ + ) + result = testdir.runpytest("--browser", "chromium") + # Body raises during call, soft assertion raises during teardown. + result.assert_outcomes(failed=1, errors=1) + out = "\n".join(result.outlines) + assert "body-fail" in out + assert "soft-fail" in out From d1b45c09ff6c65f7ddbe75312fb4110d9ee3ab2a Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Mon, 4 May 2026 15:35:13 +0200 Subject: [PATCH 2/3] refactor: lift BaseExceptionGroup import to module top --- .../pytest_playwright_asyncio/pytest_playwright.py | 12 ++++++------ .../pytest_playwright/pytest_playwright.py | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pytest-playwright-asyncio/pytest_playwright_asyncio/pytest_playwright.py b/pytest-playwright-asyncio/pytest_playwright_asyncio/pytest_playwright.py index a3cf18f..3b6454f 100644 --- a/pytest-playwright-asyncio/pytest_playwright_asyncio/pytest_playwright.py +++ b/pytest-playwright-asyncio/pytest_playwright_asyncio/pytest_playwright.py @@ -37,6 +37,12 @@ ) import pytest + +if sys.version_info >= (3, 11): + _BaseExceptionGroup = BaseExceptionGroup # noqa: F821 +else: + from exceptiongroup import BaseExceptionGroup as _BaseExceptionGroup + from playwright._impl._assertions import _soft_scope from playwright.async_api import ( Browser, @@ -90,12 +96,6 @@ def delete_output_dir(pytestconfig: Any) -> None: shutil.rmtree(entry) -if sys.version_info >= (3, 11): - _BaseExceptionGroup = BaseExceptionGroup # noqa: F821 -else: - from exceptiongroup import BaseExceptionGroup as _BaseExceptionGroup - - @pytest.fixture(autouse=True) def _playwright_soft_assertions() -> Generator[None, None, None]: with _soft_scope() as errors: diff --git a/pytest-playwright/pytest_playwright/pytest_playwright.py b/pytest-playwright/pytest_playwright/pytest_playwright.py index d68ae1c..5341090 100644 --- a/pytest-playwright/pytest_playwright/pytest_playwright.py +++ b/pytest-playwright/pytest_playwright/pytest_playwright.py @@ -35,6 +35,12 @@ ) import pytest + +if sys.version_info >= (3, 11): + _BaseExceptionGroup = BaseExceptionGroup # noqa: F821 +else: + from exceptiongroup import BaseExceptionGroup as _BaseExceptionGroup + from playwright._impl._assertions import _soft_scope from playwright.sync_api import ( Browser, @@ -87,12 +93,6 @@ def delete_output_dir(pytestconfig: Any) -> None: shutil.rmtree(entry) -if sys.version_info >= (3, 11): - _BaseExceptionGroup = BaseExceptionGroup # noqa: F821 -else: - from exceptiongroup import BaseExceptionGroup as _BaseExceptionGroup - - @pytest.fixture(autouse=True) def _playwright_soft_assertions() -> Generator[None, None, None]: with _soft_scope() as errors: From fd622265a2541844a6bcb5df2603dfdc7328f124 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Tue, 12 May 2026 09:37:25 +0200 Subject: [PATCH 3/3] feat: report soft assertion failures in call phase Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../pytest_playwright.py | 24 ++++++++++++++--- .../pytest_playwright/pytest_playwright.py | 24 ++++++++++++++--- tests/test_asyncio.py | 27 +++++++++++++++++-- tests/test_sync.py | 8 +++--- 4 files changed, 69 insertions(+), 14 deletions(-) diff --git a/pytest-playwright-asyncio/pytest_playwright_asyncio/pytest_playwright.py b/pytest-playwright-asyncio/pytest_playwright_asyncio/pytest_playwright.py index 3b6454f..50966c2 100644 --- a/pytest-playwright-asyncio/pytest_playwright_asyncio/pytest_playwright.py +++ b/pytest-playwright-asyncio/pytest_playwright_asyncio/pytest_playwright.py @@ -43,7 +43,10 @@ else: from exceptiongroup import BaseExceptionGroup as _BaseExceptionGroup -from playwright._impl._assertions import _soft_scope +try: + from playwright._impl._assertions import _soft_scope +except ImportError: + _soft_scope = None from playwright.async_api import ( Browser, BrowserContext, @@ -96,12 +99,25 @@ def delete_output_dir(pytestconfig: Any) -> None: shutil.rmtree(entry) -@pytest.fixture(autouse=True) -def _playwright_soft_assertions() -> Generator[None, None, None]: - with _soft_scope() as errors: +@pytest.hookimpl(wrapper=True, tryfirst=True) +def pytest_runtest_call(item: Any) -> Generator[None, Any, None]: + if _soft_scope is None: yield + return + hard_failure: Optional[BaseException] = None + with _soft_scope() as errors: + try: + yield + except BaseException as exc: + hard_failure = exc if not errors: + if hard_failure is not None: + raise hard_failure return + if hard_failure is not None: + raise _BaseExceptionGroup( + "Test and soft assertion failures", [hard_failure, *errors] + ) if len(errors) == 1: raise errors[0] raise _BaseExceptionGroup("Soft assertion failures", errors) diff --git a/pytest-playwright/pytest_playwright/pytest_playwright.py b/pytest-playwright/pytest_playwright/pytest_playwright.py index 5341090..119a535 100644 --- a/pytest-playwright/pytest_playwright/pytest_playwright.py +++ b/pytest-playwright/pytest_playwright/pytest_playwright.py @@ -41,7 +41,10 @@ else: from exceptiongroup import BaseExceptionGroup as _BaseExceptionGroup -from playwright._impl._assertions import _soft_scope +try: + from playwright._impl._assertions import _soft_scope +except ImportError: + _soft_scope = None from playwright.sync_api import ( Browser, BrowserContext, @@ -93,12 +96,25 @@ def delete_output_dir(pytestconfig: Any) -> None: shutil.rmtree(entry) -@pytest.fixture(autouse=True) -def _playwright_soft_assertions() -> Generator[None, None, None]: - with _soft_scope() as errors: +@pytest.hookimpl(wrapper=True, tryfirst=True) +def pytest_runtest_call(item: Any) -> Generator[None, Any, None]: + if _soft_scope is None: yield + return + hard_failure: Optional[BaseException] = None + with _soft_scope() as errors: + try: + yield + except BaseException as exc: + hard_failure = exc if not errors: + if hard_failure is not None: + raise hard_failure return + if hard_failure is not None: + raise _BaseExceptionGroup( + "Test and soft assertion failures", [hard_failure, *errors] + ) if len(errors) == 1: raise errors[0] raise _BaseExceptionGroup("Soft assertion failures", errors) diff --git a/tests/test_asyncio.py b/tests/test_asyncio.py index 2494936..7531e8d 100644 --- a/tests/test_asyncio.py +++ b/tests/test_asyncio.py @@ -1106,7 +1106,7 @@ async def test_soft(page): """ ) result = testdir.runpytest("--browser", "chromium") - result.assert_outcomes(passed=1, errors=1) + result.assert_outcomes(failed=1) assert any("goodbye" in line for line in result.outlines) @@ -1126,7 +1126,7 @@ async def test_soft(page): """ ) result = testdir.runpytest("--browser", "chromium") - result.assert_outcomes(passed=1, errors=1) + result.assert_outcomes(failed=1) out = "\n".join(result.outlines) assert "first" in out and "second" in out @@ -1145,3 +1145,26 @@ async def test_soft(page): ) result = testdir.runpytest("--browser", "chromium") result.assert_outcomes(passed=1) + + +def test_soft_assertion_does_not_shadow_body_failure( + testdir: pytest.Testdir, +) -> None: + testdir.makepyfile( + """ + import pytest + from playwright.async_api import expect + + @pytest.mark.asyncio + async def test_soft(page): + await page.set_content("
hello
") + await expect.soft(page.locator("div")).to_have_text("soft-fail") + raise RuntimeError("body-fail") + """ + ) + result = testdir.runpytest("--browser", "chromium") + # Body and soft assertion failures are grouped in call phase. + result.assert_outcomes(failed=1) + out = "\n".join(result.outlines) + assert "body-fail" in out + assert "soft-fail" in out diff --git a/tests/test_sync.py b/tests/test_sync.py index 59f6733..a5ff586 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -1090,7 +1090,7 @@ def test_soft(page): """ ) result = testdir.runpytest("--browser", "chromium") - result.assert_outcomes(passed=1, errors=1) + result.assert_outcomes(failed=1) assert any("goodbye" in line for line in result.outlines) @@ -1108,7 +1108,7 @@ def test_soft(page): """ ) result = testdir.runpytest("--browser", "chromium") - result.assert_outcomes(passed=1, errors=1) + result.assert_outcomes(failed=1) out = "\n".join(result.outlines) assert "first" in out and "second" in out @@ -1141,8 +1141,8 @@ def test_soft(page): """ ) result = testdir.runpytest("--browser", "chromium") - # Body raises during call, soft assertion raises during teardown. - result.assert_outcomes(failed=1, errors=1) + # Body and soft assertion failures are grouped in call phase. + result.assert_outcomes(failed=1) out = "\n".join(result.outlines) assert "body-fail" in out assert "soft-fail" in out