From 2c0ee7b5030d7727407a25bcfd26ebb8a0701403 Mon Sep 17 00:00:00 2001 From: Enrico Minack Date: Mon, 13 Jan 2025 11:24:54 +0100 Subject: [PATCH 1/3] Add more is_* properties to JUnit and XUnit2 TestCase - Add is_failure and is_error to JUnit TestCase - Add is_rerun and is_flaky to XUnit2 TestCase - Provide access to XUnit2 interim (rerun & flaky) results Co-authored-by: Jan Wille --- junitparser/junitparser.py | 31 ++++++++------ junitparser/xunit2.py | 56 ++++++++++++++++-------- tests/test_general.py | 14 ++++++ tests/test_xunit2.py | 88 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 155 insertions(+), 34 deletions(-) diff --git a/junitparser/junitparser.py b/junitparser/junitparser.py index 5e70e66..0f7a945 100644 --- a/junitparser/junitparser.py +++ b/junitparser/junitparser.py @@ -312,6 +312,9 @@ class TestCase(Element): time = FloatAttr() __test__ = False + # JUnit TestCase children are final results, SystemOut and SystemErr + ITER_TYPES = {t._tag: t for t in {Failure, Error, Skipped, SystemOut, SystemErr}} + def __init__(self, name: str = None, classname: str = None, time: float = None): super().__init__(self._tag) if name is not None: @@ -325,11 +328,9 @@ def __hash__(self): return super().__hash__() def __iter__(self) -> Iterator[Union[Result, System]]: - all_types = {Failure, Error, Skipped, SystemOut, SystemErr} for elem in self._elem.iter(): - for entry_type in all_types: - if elem.tag == entry_type._tag: - yield entry_type.fromelem(elem) + if elem.tag in self.ITER_TYPES: + yield self.ITER_TYPES[elem.tag].fromelem(elem) def __eq__(self, other): # TODO: May not work correctly if unreliable hash method is used. @@ -340,23 +341,25 @@ def is_passed(self): """Whether this testcase was a success (i.e. if it isn't skipped, failed, or errored).""" return not self.result + @property + def is_failure(self): + """Whether this testcase failed.""" + return any(isinstance(r, Failure) for r in self.result) + + @property + def is_error(self): + """Whether this testcase errored.""" + return any(isinstance(r, Error) for r in self.result) + @property def is_skipped(self): """Whether this testcase was skipped.""" - for r in self.result: - if isinstance(r, Skipped): - return True - return False + return any(isinstance(r, Skipped) for r in self.result) @property def result(self) -> List[FinalResult]: """A list of :class:`Failure`, :class:`Skipped`, or :class:`Error` objects.""" - results = [] - for entry in self: - if isinstance(entry, FinalResult): - results.append(entry) - - return results + return [entry for entry in self if isinstance(entry, FinalResult)] @result.setter def result(self, value: Union[FinalResult, List[FinalResult]]): diff --git a/junitparser/xunit2.py b/junitparser/xunit2.py index e77576b..8b0194e 100644 --- a/junitparser/xunit2.py +++ b/junitparser/xunit2.py @@ -12,11 +12,10 @@ There may be many others that I'm not aware of. """ -from typing import List, TypeVar +import itertools +from typing import Iterator, List, Type, TypeVar from . import junitparser -T = TypeVar("T") - class StackTrace(junitparser.System): _tag = "stackTrace" @@ -98,31 +97,54 @@ class FlakyError(InterimResult): _tag = "flakyError" +RERUN_RESULTS = {RerunFailure, RerunError, FlakyFailure, FlakyError} + + +R = TypeVar("R", bound=InterimResult) + + class TestCase(junitparser.TestCase): group = junitparser.Attr() - def _rerun_results(self, _type: T) -> List[T]: - elems = self.iterchildren(_type) - results = [] - for elem in elems: - results.append(_type.fromelem(elem)) - return results + # XUnit2 TestCase children are JUnit children and rerun results + ITER_TYPES = {t._tag: t for t in list(junitparser.TestCase.ITER_TYPES.values()) + list(RERUN_RESULTS)} + + def _interim_results(self, _type: Type[R]) -> List[R]: + return [entry for entry in self if isinstance(entry, _type)] - def rerun_failures(self): + @property + def interim_result(self) -> List[InterimResult]: + """ + A list of interim results: :class:`RerunFailure`, :class:`RerunError`, :class:`FlakyFailure`, or :class:`FlakyError` objects. + This is complementary to the result property returning final results. + """ + return self._interim_results(InterimResult) + + def rerun_failures(self) -> List[RerunFailure]: """""" - return self._rerun_results(RerunFailure) + return self._interim_results(RerunFailure) - def rerun_errors(self): + def rerun_errors(self) -> List[RerunError]: """""" - return self._rerun_results(RerunError) + return self._interim_results(RerunError) - def flaky_failures(self): + def flaky_failures(self) -> List[FlakyFailure]: """""" - return self._rerun_results(FlakyFailure) + return self._interim_results(FlakyFailure) - def flaky_errors(self): + def flaky_errors(self) -> List[FlakyError]: """""" - return self._rerun_results(FlakyError) + return self._interim_results(FlakyError) + + @property + def is_rerun(self) -> bool: + """Whether this testcase is rerun, i.e., there are rerun failures or errors.""" + return any(self.rerun_failures()) or any(self.rerun_errors()) + + @property + def is_flaky(self) -> bool: + """Whether this testcase is flaky, i.e., there are flaky failures or errors.""" + return any(self.flaky_failures()) or any(self.flaky_errors()) def add_interim_result(self, result: InterimResult): """Append an interim (rerun or flaky) result to the testcase. A testcase can have multiple interim results.""" diff --git a/tests/test_general.py b/tests/test_general.py index a8ad84a..d5ea93e 100644 --- a/tests/test_general.py +++ b/tests/test_general.py @@ -693,18 +693,32 @@ def test_case_is_skipped(self): case.result = [Skipped()] assert case.is_skipped assert not case.is_passed + assert not case.is_failure + assert not case.is_error def test_case_is_passed(self): case = TestCase() case.result = [] assert not case.is_skipped assert case.is_passed + assert not case.is_failure + assert not case.is_error def test_case_is_failed(self): case = TestCase() case.result = [Failure()] assert not case.is_skipped assert not case.is_passed + assert case.is_failure + assert not case.is_error + + def test_case_is_error(self): + case = TestCase() + case.result = [Error()] + assert not case.is_skipped + assert not case.is_passed + assert not case.is_failure + assert case.is_error class Test_Properties: diff --git a/tests/test_xunit2.py b/tests/test_xunit2.py index 8cd406f..56ba956 100644 --- a/tests/test_xunit2.py +++ b/tests/test_xunit2.py @@ -1,10 +1,10 @@ -from junitparser.xunit2 import JUnitXml, TestSuite, TestCase, RerunFailure +from junitparser.xunit2 import JUnitXml, TestSuite, TestCase, RerunFailure, RerunError, FlakyFailure, FlakyError from junitparser import Failure from copy import deepcopy class Test_TestCase: - def test_case_fromstring(self): + def test_case_rerun_fromstring(self): text = """ @@ -14,29 +14,111 @@ def test_case_fromstring(self): Error del servidor Stacktrace + System out System err """ case = TestCase.fromstring(text) assert isinstance(case, TestCase) assert case.name == "testname" + assert len(case.result) == 1 assert isinstance(case.result[0], Failure) assert case.system_out == "System out" assert case.system_err == "System err" + assert case.is_passed == False + assert case.is_failure == True + assert case.is_error == False + assert case.is_skipped == False + assert case.is_rerun == True + assert case.is_flaky == False + + interim_results = case.interim_result + assert len(interim_results) == 3 + assert isinstance(interim_results[0], RerunFailure) + assert isinstance(interim_results[1], RerunFailure) + assert isinstance(interim_results[2], RerunError) + rerun_failures = case.rerun_failures() assert len(rerun_failures) == 2 + assert isinstance(rerun_failures[0], RerunFailure) assert rerun_failures[0].message == "Not found" assert rerun_failures[0].stack_trace is None assert rerun_failures[0].system_out == "No ha encontrado" assert rerun_failures[0].system_err is None + assert isinstance(rerun_failures[1], RerunFailure) assert rerun_failures[1].message == "Server error" assert rerun_failures[1].stack_trace == "Stacktrace" assert rerun_failures[1].system_out is None assert rerun_failures[1].system_err == "Error del servidor" - assert len(case.rerun_errors()) == 0 + + rerun_errors = case.rerun_errors() + assert len(rerun_errors) == 1 + assert isinstance(rerun_errors[0], RerunError) + assert rerun_errors[0].message == "Setup error" + assert rerun_errors[0].stack_trace is None + assert rerun_errors[0].system_out is None + assert rerun_errors[0].system_err is None + assert len(case.flaky_failures()) == 0 + assert len(case.flaky_errors()) == 0 + def test_case_flaky_fromstring(self): + text = """ + + No ha encontrado + + + Error del servidor + Stacktrace + + + System out + System err + """ + case = TestCase.fromstring(text) + assert case.name == "testname" + assert len(case.result) == 0 + assert case.system_out == "System out" + assert case.system_err == "System err" + assert case.is_passed == True + assert case.is_failure == False + assert case.is_error == False + assert case.is_skipped == False + assert case.is_rerun == False + assert case.is_flaky == True + + interim_results = case.interim_result + assert len(interim_results) == 3 + assert isinstance(interim_results[0], FlakyFailure) + assert isinstance(interim_results[1], FlakyFailure) + assert isinstance(interim_results[2], FlakyError) + + assert len(case.rerun_failures()) == 0 + + assert len(case.rerun_errors()) == 0 + + flaky_failures = case.flaky_failures() + assert len(flaky_failures) == 2 + assert isinstance(flaky_failures[0], FlakyFailure) + assert flaky_failures[0].message == "Not found" + assert flaky_failures[0].stack_trace is None + assert flaky_failures[0].system_out == "No ha encontrado" + assert flaky_failures[0].system_err is None + assert isinstance(flaky_failures[1], FlakyFailure) + assert flaky_failures[1].message == "Server error" + assert flaky_failures[1].stack_trace == "Stacktrace" + assert flaky_failures[1].system_out is None + assert flaky_failures[1].system_err == "Error del servidor" + + flaky_errors = case.flaky_errors() + assert len(flaky_errors) == 1 + assert isinstance(flaky_errors[0], FlakyError) + assert flaky_errors[0].message == "Setup error" + assert flaky_errors[0].stack_trace is None + assert flaky_errors[0].system_out is None + assert flaky_errors[0].system_err is None + def test_case_rerun(self): case = TestCase("testname") rerun_failure = RerunFailure("Not found", "404") From be530c121a03e77e7262750d72f4c5c1650cd947 Mon Sep 17 00:00:00 2001 From: Enrico Minack Date: Tue, 14 Jan 2025 20:26:20 +0100 Subject: [PATCH 2/3] Simplify FINAL_RESULTS and RERUN_RESULTS Co-authored-by: Jan Wille --- junitparser/junitparser.py | 2 +- junitparser/xunit2.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/junitparser/junitparser.py b/junitparser/junitparser.py index 0f7a945..88b3586 100644 --- a/junitparser/junitparser.py +++ b/junitparser/junitparser.py @@ -313,7 +313,7 @@ class TestCase(Element): __test__ = False # JUnit TestCase children are final results, SystemOut and SystemErr - ITER_TYPES = {t._tag: t for t in {Failure, Error, Skipped, SystemOut, SystemErr}} + ITER_TYPES = {t._tag: t for t in (Failure, Error, Skipped, SystemOut, SystemErr)} def __init__(self, name: str = None, classname: str = None, time: float = None): super().__init__(self._tag) diff --git a/junitparser/xunit2.py b/junitparser/xunit2.py index 8b0194e..1d959a7 100644 --- a/junitparser/xunit2.py +++ b/junitparser/xunit2.py @@ -97,17 +97,20 @@ class FlakyError(InterimResult): _tag = "flakyError" -RERUN_RESULTS = {RerunFailure, RerunError, FlakyFailure, FlakyError} - - R = TypeVar("R", bound=InterimResult) class TestCase(junitparser.TestCase): group = junitparser.Attr() - # XUnit2 TestCase children are JUnit children and rerun results - ITER_TYPES = {t._tag: t for t in list(junitparser.TestCase.ITER_TYPES.values()) + list(RERUN_RESULTS)} + # XUnit2 TestCase children are JUnit children and intermediate results + ITER_TYPES = { + t._tag: t + for t in itertools.chain( + junitparser.TestCase.ITER_TYPES.values(), + (RerunFailure, RerunError, FlakyFailure, FlakyError), + ) + } def _interim_results(self, _type: Type[R]) -> List[R]: return [entry for entry in self if isinstance(entry, _type)] From 2b9358c11111907140495ecb5dacb19f6167d95f Mon Sep 17 00:00:00 2001 From: Enrico Minack Date: Wed, 29 Jan 2025 11:48:34 +0100 Subject: [PATCH 3/3] Fix lint issues --- junitparser/xunit2.py | 5 +++-- tests/test_xunit2.py | 24 ++++++++++++------------ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/junitparser/xunit2.py b/junitparser/xunit2.py index 1d959a7..96f4ad6 100644 --- a/junitparser/xunit2.py +++ b/junitparser/xunit2.py @@ -13,7 +13,7 @@ """ import itertools -from typing import Iterator, List, Type, TypeVar +from typing import List, Type, TypeVar from . import junitparser @@ -118,7 +118,8 @@ def _interim_results(self, _type: Type[R]) -> List[R]: @property def interim_result(self) -> List[InterimResult]: """ - A list of interim results: :class:`RerunFailure`, :class:`RerunError`, :class:`FlakyFailure`, or :class:`FlakyError` objects. + A list of interim results: :class:`RerunFailure`, :class:`RerunError`, + :class:`FlakyFailure`, or :class:`FlakyError` objects. This is complementary to the result property returning final results. """ return self._interim_results(InterimResult) diff --git a/tests/test_xunit2.py b/tests/test_xunit2.py index 56ba956..6499a52 100644 --- a/tests/test_xunit2.py +++ b/tests/test_xunit2.py @@ -25,12 +25,12 @@ def test_case_rerun_fromstring(self): assert isinstance(case.result[0], Failure) assert case.system_out == "System out" assert case.system_err == "System err" - assert case.is_passed == False - assert case.is_failure == True - assert case.is_error == False - assert case.is_skipped == False - assert case.is_rerun == True - assert case.is_flaky == False + assert case.is_passed is False + assert case.is_failure is True + assert case.is_error is False + assert case.is_skipped is False + assert case.is_rerun is True + assert case.is_flaky is False interim_results = case.interim_result assert len(interim_results) == 3 @@ -81,12 +81,12 @@ def test_case_flaky_fromstring(self): assert len(case.result) == 0 assert case.system_out == "System out" assert case.system_err == "System err" - assert case.is_passed == True - assert case.is_failure == False - assert case.is_error == False - assert case.is_skipped == False - assert case.is_rerun == False - assert case.is_flaky == True + assert case.is_passed is True + assert case.is_failure is False + assert case.is_error is False + assert case.is_skipped is False + assert case.is_rerun is False + assert case.is_flaky is True interim_results = case.interim_result assert len(interim_results) == 3