diff --git a/pytest-playwright-asyncio/pytest_playwright_asyncio/pytest_playwright.py b/pytest-playwright-asyncio/pytest_playwright_asyncio/pytest_playwright.py index 2917e04..50966c2 100644 --- a/pytest-playwright-asyncio/pytest_playwright_asyncio/pytest_playwright.py +++ b/pytest-playwright-asyncio/pytest_playwright_asyncio/pytest_playwright.py @@ -37,6 +37,16 @@ ) import pytest + +if sys.version_info >= (3, 11): + _BaseExceptionGroup = BaseExceptionGroup # noqa: F821 +else: + from exceptiongroup import BaseExceptionGroup as _BaseExceptionGroup + +try: + from playwright._impl._assertions import _soft_scope +except ImportError: + _soft_scope = None from playwright.async_api import ( Browser, BrowserContext, @@ -89,6 +99,30 @@ def delete_output_dir(pytestconfig: Any) -> None: shutil.rmtree(entry) +@pytest.hookimpl(wrapper=True, tryfirst=True) +def pytest_runtest_call(item: Any) -> Generator[None, Any, None]: + if _soft_scope is None: + yield + return + hard_failure: Optional[BaseException] = None + with _soft_scope() as errors: + try: + yield + except BaseException as exc: + hard_failure = exc + if not errors: + if hard_failure is not None: + raise hard_failure + return + if hard_failure is not None: + raise _BaseExceptionGroup( + "Test and soft assertion failures", [hard_failure, *errors] + ) + if len(errors) == 1: + raise errors[0] + raise _BaseExceptionGroup("Soft assertion failures", errors) + + def pytest_generate_tests(metafunc: Any) -> None: if "browser_name" in metafunc.fixturenames: browsers = metafunc.config.option.browser or ["chromium"] diff --git a/pytest-playwright/pytest_playwright/pytest_playwright.py b/pytest-playwright/pytest_playwright/pytest_playwright.py index 058d5d0..119a535 100644 --- a/pytest-playwright/pytest_playwright/pytest_playwright.py +++ b/pytest-playwright/pytest_playwright/pytest_playwright.py @@ -35,6 +35,16 @@ ) import pytest + +if sys.version_info >= (3, 11): + _BaseExceptionGroup = BaseExceptionGroup # noqa: F821 +else: + from exceptiongroup import BaseExceptionGroup as _BaseExceptionGroup + +try: + from playwright._impl._assertions import _soft_scope +except ImportError: + _soft_scope = None from playwright.sync_api import ( Browser, BrowserContext, @@ -86,6 +96,30 @@ def delete_output_dir(pytestconfig: Any) -> None: shutil.rmtree(entry) +@pytest.hookimpl(wrapper=True, tryfirst=True) +def pytest_runtest_call(item: Any) -> Generator[None, Any, None]: + if _soft_scope is None: + yield + return + hard_failure: Optional[BaseException] = None + with _soft_scope() as errors: + try: + yield + except BaseException as exc: + hard_failure = exc + if not errors: + if hard_failure is not None: + raise hard_failure + return + if hard_failure is not None: + raise _BaseExceptionGroup( + "Test and soft assertion failures", [hard_failure, *errors] + ) + if len(errors) == 1: + raise errors[0] + raise _BaseExceptionGroup("Soft assertion failures", errors) + + def pytest_generate_tests(metafunc: Any) -> None: if "browser_name" in metafunc.fixturenames: browsers = metafunc.config.option.browser or ["chromium"] diff --git a/tests/test_asyncio.py b/tests/test_asyncio.py index 539a883..7531e8d 100644 --- a/tests/test_asyncio.py +++ b/tests/test_asyncio.py @@ -1091,3 +1091,80 @@ async def test_connect_options(page): else: os.kill(server_process.pid, signal.SIGINT) server_process.wait() + + +def test_soft_assertion_single_failure(testdir: pytest.Testdir) -> None: + testdir.makepyfile( + """ + import pytest + from playwright.async_api import expect + + @pytest.mark.asyncio + async def test_soft(page): + await page.set_content("
hello
") + await expect.soft(page.locator("div")).to_have_text("goodbye") + """ + ) + result = testdir.runpytest("--browser", "chromium") + result.assert_outcomes(failed=1) + assert any("goodbye" in line for line in result.outlines) + + +def test_soft_assertion_multiple_failures_exception_group( + testdir: pytest.Testdir, +) -> None: + testdir.makepyfile( + """ + import pytest + from playwright.async_api import expect + + @pytest.mark.asyncio + async def test_soft(page): + await page.set_content("
hello
") + await expect.soft(page.locator("div")).to_have_text("first") + await expect.soft(page.locator("div")).to_have_text("second") + """ + ) + result = testdir.runpytest("--browser", "chromium") + result.assert_outcomes(failed=1) + out = "\n".join(result.outlines) + assert "first" in out and "second" in out + + +def test_soft_assertion_passes_when_all_match(testdir: pytest.Testdir) -> None: + testdir.makepyfile( + """ + import pytest + from playwright.async_api import expect + + @pytest.mark.asyncio + async def test_soft(page): + await page.set_content("
hello
") + await expect.soft(page.locator("div")).to_have_text("hello") + """ + ) + result = testdir.runpytest("--browser", "chromium") + result.assert_outcomes(passed=1) + + +def test_soft_assertion_does_not_shadow_body_failure( + testdir: pytest.Testdir, +) -> None: + testdir.makepyfile( + """ + import pytest + from playwright.async_api import expect + + @pytest.mark.asyncio + async def test_soft(page): + await page.set_content("
hello
") + await expect.soft(page.locator("div")).to_have_text("soft-fail") + raise RuntimeError("body-fail") + """ + ) + result = testdir.runpytest("--browser", "chromium") + # Body and soft assertion failures are grouped in call phase. + result.assert_outcomes(failed=1) + out = "\n".join(result.outlines) + assert "body-fail" in out + assert "soft-fail" in out diff --git a/tests/test_sync.py b/tests/test_sync.py index 374d919..a5ff586 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -1077,3 +1077,72 @@ def test_connect_options(page): else: os.kill(server_process.pid, signal.SIGINT) server_process.wait() + + +def test_soft_assertion_single_failure(testdir: pytest.Testdir) -> None: + testdir.makepyfile( + """ + from playwright.sync_api import expect + + def test_soft(page): + page.set_content("
hello
") + expect.soft(page.locator("div")).to_have_text("goodbye") + """ + ) + result = testdir.runpytest("--browser", "chromium") + result.assert_outcomes(failed=1) + assert any("goodbye" in line for line in result.outlines) + + +def test_soft_assertion_multiple_failures_exception_group( + testdir: pytest.Testdir, +) -> None: + testdir.makepyfile( + """ + from playwright.sync_api import expect + + def test_soft(page): + page.set_content("
hello
") + expect.soft(page.locator("div")).to_have_text("first") + expect.soft(page.locator("div")).to_have_text("second") + """ + ) + result = testdir.runpytest("--browser", "chromium") + result.assert_outcomes(failed=1) + out = "\n".join(result.outlines) + assert "first" in out and "second" in out + + +def test_soft_assertion_passes_when_all_match(testdir: pytest.Testdir) -> None: + testdir.makepyfile( + """ + from playwright.sync_api import expect + + def test_soft(page): + page.set_content("
hello
") + expect.soft(page.locator("div")).to_have_text("hello") + """ + ) + result = testdir.runpytest("--browser", "chromium") + result.assert_outcomes(passed=1) + + +def test_soft_assertion_does_not_shadow_body_failure( + testdir: pytest.Testdir, +) -> None: + testdir.makepyfile( + """ + from playwright.sync_api import expect + + def test_soft(page): + page.set_content("
hello
") + expect.soft(page.locator("div")).to_have_text("soft-fail") + raise RuntimeError("body-fail") + """ + ) + result = testdir.runpytest("--browser", "chromium") + # Body and soft assertion failures are grouped in call phase. + result.assert_outcomes(failed=1) + out = "\n".join(result.outlines) + assert "body-fail" in out + assert "soft-fail" in out