From f5bae75bdd607dccabbaea75488c6c7220bd0cd4 Mon Sep 17 00:00:00 2001 From: MarvKler <98239503+MarvKler@users.noreply.github.com> Date: Wed, 5 Nov 2025 18:25:49 +0100 Subject: [PATCH 1/8] fixed configuration & resetting of log levels (#1) * set loglevel correctly * gitignore --------- Co-authored-by: Marvin Klerx --- .gitignore | 4 ++++ atest/01_SimpleTestSuite.robot | 13 +++++++++++++ src/RetryFailed/retry_failed.py | 4 ++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index ea1a214..b5f4a53 100644 --- a/.gitignore +++ b/.gitignore @@ -129,3 +129,7 @@ dmypy.json .pyre/ atest/results results +log.html +output.xml +report.html +.vscode/launch.json diff --git a/atest/01_SimpleTestSuite.robot b/atest/01_SimpleTestSuite.robot index 1f1a194..9c50175 100755 --- a/atest/01_SimpleTestSuite.robot +++ b/atest/01_SimpleTestSuite.robot @@ -10,27 +10,40 @@ ${retry_2} ${0} *** Test Cases *** My Simple Test Log Hello World + Log This TRACE message should not be logged! level=TRACE Should Be Equal Hello Hello Sometime Fail [Tags] test:retry(1) + Log This TRACE message can be logged sometimes level=TRACE Should Be True ${{random.randint(0, 1)}} == 1 Sometime Fail1 [Tags] test:retry(3) + Log This TRACE message can be logged sometimes level=TRACE Should Be True ${{random.randint(0, 1)}} == 1 +My Simple Test2 + Log Hello World + Log This TRACE message should not be logged! level=TRACE + Should Be Equal Hello Hello + +Log Trace Message + Log This TRACE message should not be logged! level=TRACE + Sometime Fail2 [Tags] test:retry(1) Should Be True ${{random.randint(0, 1)}} == 1 Passes after 3 Fails [Tags] test:retry(3) + Log This TRACE message should be logged on failures only! level=TRACE Should Be Equal ${retry_1} ${3} [Teardown] Set Suite Variable ${retry_1} ${retry_1 + 1} Fails on 4th Exec [Tags] task:retry(5) + Log This TRACE message should be logged on failures only! level=TRACE Should Be Equal ${retry_2} ${4} [Teardown] Set Suite Variable ${retry_2} ${retry_2 + 1} diff --git a/src/RetryFailed/retry_failed.py b/src/RetryFailed/retry_failed.py index e31013b..f8038ff 100644 --- a/src/RetryFailed/retry_failed.py +++ b/src/RetryFailed/retry_failed.py @@ -44,7 +44,7 @@ def start_test(self, test, result): if self.retries: BuiltIn().set_test_variable("${RETRYFAILED_RETRY_INDEX}", self.retries) if self.log_level is not None: - self._original_log_level = BuiltIn()._context.output.set_log_level(self.log_level) + self._original_log_level = BuiltIn().set_log_level(self.log_level) for tag in test.tags: retry_match = re.match(r"(?:test|task):retry\((\d+)\)", tag) if retry_match: @@ -55,7 +55,7 @@ def start_test(self, test, result): def end_test(self, test, result): if self.retries and self._original_log_level is not None: - BuiltIn()._context.output.set_log_level(self._original_log_level) + BuiltIn().set_log_level(self._original_log_level) if not self.max_retries: self.retries = 0 return From 2b03f358c1e9f7ec8cd9d70986123b40c8f21f0f Mon Sep 17 00:00:00 2001 From: MarvKler <98239503+MarvKler@users.noreply.github.com> Date: Thu, 6 Nov 2025 14:25:02 +0100 Subject: [PATCH 2/8] Added keyword retry listener (#2) * v1 keywordretrylistener * v1 keywordretrylistener * retry kewords if failed * set loglevel for keyword retry * docs * docs * tests * removed status param * removed not required imports --------- Co-authored-by: Marvin Klerx --- .gitignore | 1 + README.md | 94 ++++++++++++---- atest/01_SimpleTestSuite.robot | 26 ++--- atest/02_KeywordRetryListener.robot | 81 ++++++++++++++ atest/run_atest.sh | 2 +- src/RetryFailed/retry_failed.py | 167 ++++++++++++++++++++++++++-- 6 files changed, 325 insertions(+), 46 deletions(-) create mode 100644 atest/02_KeywordRetryListener.robot diff --git a/.gitignore b/.gitignore index b5f4a53..5622e95 100644 --- a/.gitignore +++ b/.gitignore @@ -133,3 +133,4 @@ log.html output.xml report.html .vscode/launch.json +temp_listener.py diff --git a/README.md b/README.md index 0f91b9d..1778b35 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,25 @@ # robotframework-retryfailed -A listener to automatically retry tests or tasks based on tags. +A listener to automatically retry tests, tasks or keywords based on tags. ## Installation Install with pip: +``` +pip install robotframework-retryfailed +``` - pip install robotframework-retryfailed +## CLI Arguments + +You can configure the following CLI arguments when registering the listener in your robotframework cli call: + +| Argument | Description | Mandatory | Default Value | +|----------|-------------|-----------|---------------| +| ``global_test_retries`` | Define a global number of retries which is valid for ALL your tests by default! | No | `**0** | +| ``keep_retried_tests`` | Define if the retried tests should be kept in the logs or not. If ``True``, they will be marked with status ``Skip`` | No | **False** | +| ``log_level`` | If set, the loglevel will be changed to the given value IF a test / keyword is getting retried. | No | **None** | +| ``warn_on_test_retry`` | If ``True``, the retried tests will be logged as warning to the ``log.html`` | No | **True** | +| ``warn_on_kw_retry`` | If ``True``, the retried keywords will be logged as warning to the ``log.html`` | No | **False** | ## Usage @@ -14,27 +27,54 @@ Add the listener to your robot execution, via command line arguments. When your tests do fail and you have tagged them with `test:retry(2)`, it will retry the test 2 times. Retry can be also set globally as a parameter to the listener. -### Attaching Listener +### Attaching Listener - Retry Tests Example: +``` +# Attaching listener without any default retry configuratio +robot --listener RetryFailed - robot --listener RetryFailed +# Attach listener & retrying every failed tests once if failed +robot --listener RetryFailed:1 - robot --listener RetryFailed:1 +# Attaching listener, retrying every tests once, keep failed tests in logfile & increase loglevel to TRACE for retry tests. +robot --listener RetryFailed:1:True:TRACE +``` -Second one will by default retry once every failing test. +### Retry Test Cases - Tagging Tests -### Tagging Tests - -Example: +If attaching the listener without any default retry configuration, you must set the count of max. retry as ``Test Tag``. - *** Test Cases *** - Test Case - [Tags] test:retry(2) - Log This test will be retried 2 times if it fails +See Example: +``` +*** Test Cases *** +Test Case + [Tags] test:retry(2) + Log This test will be retried 2 times if it fails +``` Tagging tasks by `task:retry(3)` should also work. +### Retry Keywords - Tagging Keywords + +The ``KeywordRetryListener`` is basically always ``activated`` & needs to be defined by yourself within your test. + +It makes no sense to configure a default retry count for every keyword, because usually too many keywords are part a test / parent keyword to retry them **ALL** ! +This means, you need to define the keywords which should be retried by yourself! + +Therefore, you must configure the following ``Keyword Tags``: +``` +*** Keywords *** +KeywordABC + [Documentation] Keyword takes max. 1 retry! + [Tags] keyword:retry(1) + +KeywordDEF + [Documentation] Keyword takes max. 5 retries! + [Tags] keyword:retry(5) +``` + + ### Configuration On top of specifying the number of retries, you can also define whether your want to **keep the logs** of the retried tests and change the **log level** when retrying, by providing respectfully second and third parameter to the listener: `RetryFailed:::` @@ -42,12 +82,28 @@ On top of specifying the number of retries, you can also define whether your wan By default the logs of the retried tests are not kept, and the log level remains the same as the regular run. Example: +``` +# keep the logs of the retried tests +robot --listener RetryFailed:1:True + +# does not keep the logs of the retried tests and change log level to DEBUG when retrying +robot --listener RetryFailed:2:False:DEBUG + +# keep the logs of the retried tests and change the log level to TRACE when retrying +robot --listener RetryFailed:1:True:TRACE + +# Same like previous one, but keep in mind: all retried tests are getting logged as warning. But all retried keywords are not getting logged as warning! +robot --listener RetryFailed:1:True:TRACE + +# Both retried tests & retried keywords are getting logged as warning into the log.html +robot --listener RetryFailed:1:True:TRACE:True:True + +# Only retried keywords are getting logged as warning into the log.html +robot --listener RetryFailed:1:True:TRACE:False:True +``` - # keep the logs of the retried tests - robot --listener RetryFailed:1:True +### Log Warnings at Retry - # does not keep the logs of the retried tests and change log level to DEBUG when retrying - robot --listener RetryFailed:2:False:DEBUG +If you've set both parameters, ``warn_on_test_retry`` & ``warn_on_kw_retry``, to ``False``, a simple ``Info`` message gets logged during the keyword execution in the log.html. - # keep the logs of the retried tests and change the log level to TRACE when retrying - robot --listener RetryFailed:1:True:TRACE +You won't see at the top of the log file, but you can find it directly within the logged keyword execution. \ No newline at end of file diff --git a/atest/01_SimpleTestSuite.robot b/atest/01_SimpleTestSuite.robot index 9c50175..ea8312c 100755 --- a/atest/01_SimpleTestSuite.robot +++ b/atest/01_SimpleTestSuite.robot @@ -1,8 +1,6 @@ -*** Settings *** -# Library RetryFailed log_level=TRACE - - *** Variables *** +${tc_01} ${0} +${tc_02} ${0} ${retry_1} ${0} ${retry_2} ${0} @@ -13,15 +11,17 @@ My Simple Test Log This TRACE message should not be logged! level=TRACE Should Be Equal Hello Hello -Sometime Fail - [Tags] test:retry(1) +TC01 - Retry Once + [Tags] test:retry(2) Log This TRACE message can be logged sometimes level=TRACE - Should Be True ${{random.randint(0, 1)}} == 1 + VAR ${tc_01} = ${${tc_01}+1} scope=SUITE + Should Be Equal As Integers ${tc_01} ${2} -Sometime Fail1 - [Tags] test:retry(3) +TC01 - Retry Twice + [Tags] test:retry(2) Log This TRACE message can be logged sometimes level=TRACE - Should Be True ${{random.randint(0, 1)}} == 1 + VAR ${tc_02} = ${${tc_02}+1} scope=SUITE + Should Be Equal As Integers ${tc_02} ${3} My Simple Test2 Log Hello World @@ -31,17 +31,13 @@ My Simple Test2 Log Trace Message Log This TRACE message should not be logged! level=TRACE -Sometime Fail2 - [Tags] test:retry(1) - Should Be True ${{random.randint(0, 1)}} == 1 - Passes after 3 Fails [Tags] test:retry(3) Log This TRACE message should be logged on failures only! level=TRACE Should Be Equal ${retry_1} ${3} [Teardown] Set Suite Variable ${retry_1} ${retry_1 + 1} -Fails on 4th Exec +Passes on 4th Exec [Tags] task:retry(5) Log This TRACE message should be logged on failures only! level=TRACE Should Be Equal ${retry_2} ${4} diff --git a/atest/02_KeywordRetryListener.robot b/atest/02_KeywordRetryListener.robot new file mode 100644 index 0000000..8590363 --- /dev/null +++ b/atest/02_KeywordRetryListener.robot @@ -0,0 +1,81 @@ +*** Variables *** +${counter_01: int} = 0 +${counter_02: int} = 0 +${counter_03: int} = 0 + + +*** Test Cases *** +Test 1 - PASS + [Tags] test:retry(0) + Successful Keyword + +Test 2 - Failed - Pass on 2. Retry + [Tags] test:retry(0) + Error - Pass on Retry 2 + +Test 3 - Failed Child Keyword - Passed on Multiple Retries + [Tags] test:retry(0) + VAR ${counter_01} = ${0} scope=suite + + Test 3 - Failed Child Keyword + +Test 4 - Failed - Retry Test & Keywords + [Tags] test:retry(10) + + Test 4 - Parent Keyword + +Test 5 - Failed Multiple Child Keywords - Pass on Retry + [Tags] test:retry(2) + + Test 5 - Parent Keyword + + +*** Keywords *** +Successful Keyword + [Tags] keyword:retry(2) + Log Successful + +Error - Pass on Retry + [Arguments] ${successful_retry: int} + [Tags] keyword:retry(2) + ${counter_01} = Evaluate $counter_01 + 1 + VAR ${counter_01} = ${counter_01} scope=SUITE + Should Be Equal ${counter_01} ${successful_retry} + +Test 3 - Failed Child Keyword + [Tags] keyword:retry(3) + + Error - Pass on Retry 5 + +Test 4 - Parent Keyword + [Tags] keyword:retry(2) + + Test 4 - Child Keyword 20 + +Test 4 - Child Keyword + [Arguments] ${successful_retry: int} + [Tags] keyword:retry(2) + ${counter_02} = Evaluate $counter_02 + 1 + VAR ${counter_02} = ${counter_02} scope=GLOBAL + Should Be Equal ${counter_02} ${successful_retry} + +Test 5 - Parent Keyword + [Tags] keyword:retry(2) + + Test 5 - First Child Keyword + +Test 5 - First Child Keyword + [Tags] keyword:retry(2) + + Test 5 - Second Child Keyword 70 + +Test 5 - Second Child Keyword + [Arguments] ${successful_retry: int} + [Tags] keyword:retry(2) + ${counter_03} = Evaluate $counter_03 + 1 + VAR ${counter_03} = ${counter_03} scope=GLOBAL + Should Be Equal ${counter_03} ${successful_retry} + + + + diff --git a/atest/run_atest.sh b/atest/run_atest.sh index 171757c..42e06b1 100755 --- a/atest/run_atest.sh +++ b/atest/run_atest.sh @@ -1 +1 @@ -robot -d results --listener RetryFailed 01_SimpleTestSuite.robot \ No newline at end of file +robot -d results --listener RetryFailed:10:True:TRACE 01_SimpleTestSuite.robot \ No newline at end of file diff --git a/src/RetryFailed/retry_failed.py b/src/RetryFailed/retry_failed.py index f8038ff..53f83cf 100644 --- a/src/RetryFailed/retry_failed.py +++ b/src/RetryFailed/retry_failed.py @@ -12,39 +12,78 @@ See the License for the specific language governing permissions and limitations under the License.""" +import copy import re -from robot.api import ExecutionResult, ResultVisitor, logger +from dataclasses import dataclass +from pathlib import Path + +from robot.api import ExecutionResult, ResultVisitor from robot.api.deco import library from robot.libraries.BuiltIn import BuiltIn from robot.utils.robottypes import is_truthy +from robot.result import Keyword as ResultKeyword +from robot.running import Keyword as RunningKeyword +from robot.running import TestCase as RunningTestCase +from robot.result import TestCase as ResultTestCase duplicate_test_pattern = re.compile( r"Multiple .*? with name '(?P.*?)' executed in.*? suite '(?P.*?)'." ) linebreak = "\n" +@dataclass +class KeywordMetaData: + kw_obj: RunningKeyword + kw_index: int + kw_name: str + kw_source: str + kw_lineno: int + retries: int + retries_performed: int + + @library(scope="GLOBAL") class RetryFailed: ROBOT_LISTENER_API_VERSION = 3 - def __init__(self, global_retries=0, keep_retried_tests=False, log_level=None): + def __init__(self, + global_test_retries: int = 0, + keep_retried_tests: bool = False, + log_level: str | None = None, + warn_on_test_retry: bool = True, + warn_on_kw_retry: bool = False + ): self.ROBOT_LIBRARY_LISTENER = self + + # Generic Settings + self.warn_on_test_retry: bool = is_truthy(warn_on_test_retry) + self.warn_on_kw_retry: bool = is_truthy(warn_on_kw_retry) + + # TestRetryListener self.retried_tests = [] self.retries = 0 - self._max_retries_by_default = int(global_retries) - self.max_retries = global_retries + self._max_retries_by_default = int(global_test_retries) + self.max_retries = global_test_retries self.keep_retried_tests = is_truthy(keep_retried_tests) self.log_level = log_level self._original_log_level = None + self.test_retry_active: bool = False + self.original_testcase_object: RunningTestCase = None - def start_test(self, test, result): + # KeywordRetryListener + self.retry_keywords: list[KeywordMetaData] = [] + self._index_counter: int = 1 + + def start_test(self, test: RunningTestCase, result: ResultTestCase): if self.retries: BuiltIn().set_test_variable("${RETRYFAILED_RETRY_INDEX}", self.retries) if self.log_level is not None: self._original_log_level = BuiltIn().set_log_level(self.log_level) + if self.retries == 0 and not self.test_retry_active: + self.original_testcase_object = copy.deepcopy(test) for tag in test.tags: retry_match = re.match(r"(?:test|task):retry\((\d+)\)", tag) if retry_match: @@ -53,7 +92,7 @@ def start_test(self, test, result): self.max_retries = self._max_retries_by_default return - def end_test(self, test, result): + def end_test(self, test: RunningTestCase, result: ResultTestCase): if self.retries and self._original_log_level is not None: BuiltIn().set_log_level(self._original_log_level) if not self.max_retries: @@ -61,25 +100,28 @@ def end_test(self, test, result): return if result.status == "FAIL": if self.retries < self.max_retries: + self.test_retry_active = True index = test.parent.tests.index(test) - test.parent.tests.insert(index + 1, test) + test.parent.tests.insert(index + 1, copy.deepcopy(self.original_testcase_object)) result.status = "SKIP" result.message += "\nSkipped for Retry" self.retried_tests.append(test.longname) self.retries += 1 return else: + self.test_retry_active = False result.message += ( f"{linebreak * bool(result.message)}[RETRY] FAIL on {self.retries}. retry." ) else: if self.retries: + self.test_retry_active = False result.message += ( f"{linebreak * bool(result.message)}[RETRY] PASS on {self.retries}. retry." ) self.retries = 0 return - + def end_suite(self, suite, result): test_dict = {} result_dict = {} @@ -89,6 +131,93 @@ def end_suite(self, suite, result): result.tests = list(result_dict.values()) suite.tests = list(test_dict.values()) + def start_keyword(self, keyword: RunningKeyword, result: ResultKeyword): + + for tag in result.tags: + retry_kw = re.match(r"keyword:retry\((\d+)\)", tag) + if not retry_kw: + return + _retries = int(retry_kw.group(1)) + if retry_kw and _retries > 0: + kw_data = KeywordMetaData ( + kw_obj = keyword, + kw_index = keyword.parent.body.index(keyword), + kw_name = keyword.name, + kw_source = Path(keyword.source).name, + kw_lineno = keyword.lineno, + retries = _retries, + retries_performed = 0, + ) + + # check if keyword is already registered for the retry + for registered_retry_keyword in self.retry_keywords: + if ( + registered_retry_keyword.kw_name == kw_data.kw_name + and + registered_retry_keyword.kw_source == kw_data.kw_source + and + registered_retry_keyword.kw_lineno == kw_data.kw_lineno + ): + return + self.retry_keywords.append(kw_data) + + def end_keyword(self, keyword: RunningKeyword, result: ResultKeyword): + + executed_kw_name = keyword.name + executed_kw_source = Path(keyword.source).name + + # reset original loglevel + if self._original_log_level: + BuiltIn().set_log_level(self._original_log_level) + + match_kw_retry = False + kw_to_retry: KeywordMetaData + for index, kw in enumerate(self.retry_keywords): + if kw.kw_name != executed_kw_name or kw.kw_source != executed_kw_source: + continue + else: + match_kw_retry = True + current_index = index + kw_to_retry = kw + + # If currently executed keyword does not match any defined RetryKeyword -> just return + if not match_kw_retry: + return + + link = self._get_keyword_link(result) + + if result.status == "PASS": + doc = f"[Keyword: {kw_to_retry.kw_name}] PASSED on {kw_to_retry.retries_performed}. retry." + msg = f"[Keyword: {link}] PASSED on {kw_to_retry.retries_performed}. retry." + if kw_to_retry.retries_performed > 0: + level: str = "WARN" if self.warn_on_kw_retry else "INFO" + BuiltIn().log(msg, level=level, html=True) + result.doc += f"\n\n{doc}" + self.retry_keywords.pop(current_index) + + if result.status == "FAIL": + + if kw_to_retry.retries and kw_to_retry.retries_performed < kw_to_retry.retries: + # Set loglevel for retry + if self.log_level: + self._original_log_level = BuiltIn().set_log_level(self.log_level) + + keyword.parent.body.insert(kw_to_retry.kw_index + self._index_counter, kw_to_retry.kw_obj) + result.status = "NOT RUN" + kw_to_retry.retries_performed += 1 + self.retry_keywords[current_index].retries_performed = kw_to_retry.retries_performed + + msg = f"[Keyword: {kw_to_retry.kw_name}] - Skipped for {self.retry_keywords[current_index].retries_performed}. Retry..." + result.doc += f"\n\n{msg}" + else: + doc = f"{"\n\n" * bool(result.message)}[Keyword: {kw_to_retry.kw_name}] FAILED on {kw_to_retry.retries_performed}. retry." + msg = f"{"\n\n" * bool(result.message)}[Keyword: {link}] FAILED on {self.retry_keywords[current_index].retries_performed}. retry." + result.doc += doc + result.message += doc + level: str = "WARN" if self.warn_on_kw_retry else "INFO" + BuiltIn().log(msg.replace("\n", ""), level=level, html=True) + self.retry_keywords.pop(current_index) + def message(self, message): if message.level == "WARN": match = duplicate_test_pattern.match(message.message) @@ -96,17 +225,33 @@ def message(self, message): message.message = ( f"Retry {self.retries}/{self.max_retries} of test '{match.group('test')}':" ) + if not self.warn_on_test_retry: + message.level = "INFO" def output_file(self, original_output_xml): result = ExecutionResult(original_output_xml) - result.visit(RetryMerger(self.retried_tests, self.keep_retried_tests)) + result.visit(RetryMerger(self.retried_tests, self.keep_retried_tests, self.warn_on_test_retry)) result.save() + + def _get_keyword_link(self, keyword_result: ResultKeyword): + link = ( + f"' + f"{keyword_result.kwname}" + f"" + if keyword_result.id + else keyword_result.kwname + ) + return link class RetryMerger(ResultVisitor): - def __init__(self, retried_tests, keep_retried_tests=False): + def __init__(self, retried_tests, keep_retried_tests=False, warn_on_test_retry: bool = True): self.retried_tests = retried_tests self.keep_retried_tests = keep_retried_tests + self.warn_on_test_retry = warn_on_test_retry self.test_ids = {} def start_suite(self, suite): @@ -126,7 +271,7 @@ def start_errors(self, errors): messages = [] retry_messages = {} for message in errors.messages: - if message.level == "WARN": + if message.level == "WARN" and self.warn_on_test_retry: pattern = re.compile( r"Retry (?P\d+)/(?P\d+) of test '(?P.+)':" ) From 0858031dc9ae675d44ab5a45c35ff818e69b4b50 Mon Sep 17 00:00:00 2001 From: MarvKler <98239503+MarvKler@users.noreply.github.com> Date: Wed, 3 Dec 2025 19:38:22 +0100 Subject: [PATCH 3/8] refactored to pyproject, flit, ruff, etc and added tests and typing, fixed some comments (#3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactored to pyproject, flit, ruff, etc and added tests and typing * merged pr-14 to fork, fixed some of the comments --------- Co-authored-by: René Co-authored-by: Marvin Klerx --- .github/copilot-instructions.md | 34 ++++ .pre-commit-config.yaml | 30 ++++ CONTRIBUTING.md | 58 +++++++ atest/01_SimpleTestSuite.robot | 96 +++++------ atest/02_KeywordRetryListener.robot | 12 +- atest/03_KeywordRetry.robot | 66 +++++++ atest/KeywordRetry.py | 26 +++ atest/run_atest.sh | 2 +- bootstrap.py | 71 ++++++++ createPip_whl_tar.sh | 8 - pyproject.toml | 112 ++++++++++++ requirements-dev.txt | 3 - setup.py | 49 ------ src/RetryFailed/__init__.py | 2 + src/RetryFailed/retry_failed.py | 255 +++++++++++++++------------- 15 files changed, 593 insertions(+), 231 deletions(-) create mode 100644 .github/copilot-instructions.md create mode 100644 .pre-commit-config.yaml create mode 100644 CONTRIBUTING.md create mode 100644 atest/03_KeywordRetry.robot create mode 100644 atest/KeywordRetry.py create mode 100644 bootstrap.py delete mode 100755 createPip_whl_tar.sh create mode 100644 pyproject.toml delete mode 100644 requirements-dev.txt delete mode 100644 setup.py diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..d9ad27c --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,34 @@ +# robotframework-retryfailed – Copilot Guide + +## Architecture +- `src/RetryFailed/retry_failed.py` hosts the entire listener/library; `RetryFailed` exposes ROBOT_LISTENER_API_VERSION 3 and is imported via `RetryFailed` package (see `src/RetryFailed/__init__.py`). +- `KeywordMetaData` snapshots keyword name/source/line so we can reinsert the same `RunningKeyword` when retrying without duplicating registrations. +- `start_test` deep-copies the original `RunningTestCase` and sets `${RETRYFAILED_RETRY_INDEX}`; `end_test` requeues failures by inserting the saved test back into `test.parent.tests` and flagging original results as SKIP. +- `end_suite` and `RetryMerger` (a `robot.api.ResultVisitor`) rewrite the execution tree so persisted output either collapses duplicates or keeps them when `keep_retried_tests=True`. + +## Retry semantics +- Tests/tasks opt in via tags like `[Tags] test:retry(2)` or `task:retry(5)`; missing tags fall back to the constructor’s `global_test_retries` default supplied on the `--listener RetryFailed:` CLI. +- Keyword retries are entirely tag-driven: `[Tags] keyword:retry()` on either user keywords or test cases, with `start_keyword` storing metadata and `end_keyword` ensuring only one registration per (name, source, lineno). +- The listener exposes two warning toggles (`warn_on_test_retry`, `warn_on_kw_retry`) that gate whether BuiltIn logs use WARN or INFO when retries happen; preserve these semantics whenever adding new log messages. +- When `log_level` is passed, retries temporarily raise Robot’s log level; always reset through `_original_log_level` before exiting the listener hook to avoid leaking state between tests/keywords. + +## Result shaping & messaging +- `message()` intercepts duplicate-test WARNs emitted by Robot and rewrites them into retry status lines so final logs stay meaningful; any new warning flow should piggyback on this formatting pattern (`Retry {x}/{y} of test ...`). +- `output_file()` always rewrites `output.xml` in place via `ExecutionResult.visit(RetryMerger(...))`; if you touch result data structures remember that post-processing happens after Robot finishes. +- HTML links inserted by `_get_keyword_link()` and `_get_test_link()` rely on Robot-generated element ids; keep IDs intact when adjusting result objects. + +## Tests & verification +- Acceptance tests live under `atest/`; run both suites locally with `robot -d results --listener RetryFailed:10:True:TRACE atest/01_SimpleTestSuite.robot` and `atest/02_KeywordRetryListener.robot` (scripted in `atest/run_atest.sh`). +- The suites rely on stateful suite variables (`VAR … scope=SUITE/GLOBAL`), so run them serially and reset `${counter_*}` variables when writing new cases. +- There are no unit tests; regression coverage hinges on these Robot suites plus manual log inspection of `output.xml`/`log.html`. + +## Development workflow +- Use Python ≥3.8; `pip install -r requirements-dev.txt` installs the project in editable mode plus the `dev` extra (flit, mypy, ruff, robocop, check-manifest, twine). +- Packaging now runs through Flit: bump `__version__` in `src/RetryFailed/__init__.py`, then execute `./createPip_whl_tar.sh` (wraps `flit build`, `twine check`, and `flit publish`). +- `.pre-commit-config.yaml` is authoritative (ruff lint/format, mypy, robocop); the same settings live in `pyproject.toml`, so add new tooling there. + +## Contribution conventions +- Prefer pure Python stdlib + Robot APIs; new deps must be justified because the listener is imported inside Robot runs. +- Keep listener hooks fast and side-effect free—no network/file IO inside `start_*`/`end_*` without caching since they run per test/keyword. +- Document any new listener args in both the README table and `atest` tags so behavior stays discoverable. +- When adding retry metadata, key lookups currently use `(kw_name, kw_source, kw_lineno)`; preserve or extend that tuple instead of relying solely on names to avoid clashes between identically named keywords in different files. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..652e285 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,30 @@ +repos: + # mypy linter for Python files + - repo: local + hooks: + - id: mypy + name: mypy + description: 'Run mypy for static type checking' + entry: mypy + language: python + types_or: [python, pyi] + require_serial: true + + # Ruff linter for Python files + - repo: local + hooks: + - id: ruff-format + name: ruff-format + description: "Run 'ruff format' for extremely fast Python formatting" + entry: ruff format --force-exclude + language: python + types_or: [python, pyi, jupyter] + require_serial: true + + - id: ruff + name: ruff + description: "Run 'ruff' for extremely fast Python linting" + entry: ruff check --force-exclude + language: python + types_or: [python, pyi, jupyter] + require_serial: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..c9198b0 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,58 @@ +# Contributing to robotframework-retryfailed + +Thanks for helping improve the RetryFailed listener! This project is intentionally small, so the +workflow is simple and scripted. The steps below explain how to set up an environment, run +tooling, and publish a release. + +## 1. Environment setup +1. Use Python 3.8+. +2. Create a virtual environment and install the dev extras: + ```bash + python -m venv .venv + source .venv/bin/activate + pip install -r requirements-dev.txt + ``` + The `dev` extra installs flit, mypy, ruff, robocop, check-manifest, twine, and Robot Framework + add-ons used in the acceptance suites. +3. Install the pre-commit hooks (optional but recommended): + ```bash + pre-commit install + ``` + +## 2. Linting & formatting +- **Python formatting**: `ruff format .` +- **Python linting**: `ruff check .` +- **Static typing**: `mypy src` +- **Robot Framework linting**: `robocop check --include *.robot --include *.resource` +- **Robot Framework formatting**: `robocop format --include *.robot --include *.resource` +- **Pre-commit (full suite)**: `pre-commit run --all-files` + +The same settings live in `pyproject.toml`, so CI and local runs stay aligned. Prefer running the +standalone commands when iterating quickly, and fall back to the pre-commit aggregate before +delivering work. + +## 3. Tests +All regression coverage is provided by the acceptance suites under `atest/`. Run them from the +repository root with the listener installed in editable mode: +```bash +robot -d results --listener RetryFailed:10:True:TRACE atest/01_SimpleTestSuite.robot +robot -d results --listener RetryFailed:10:True:TRACE atest/02_KeywordRetryListener.robot +``` +Inspect `results/output.xml` and `results/log.html` to confirm retries are recorded correctly. + +## 4. Release workflow +1. Bump `__version__` inside `src/RetryFailed/__init__.py`. +2. Verify the manifest and build artifacts: + ```bash + check-manifest --update + flit build + twine check dist/* + ``` +3. Upload to PyPI (after any manual smoke tests): + ```bash + flit publish + ``` + The helper script `./createPip_whl_tar.sh` automates the same sequence with a keypress between + verification and upload. + +Please open an issue before large or breaking changes so we can align on direction. Happy hacking! diff --git a/atest/01_SimpleTestSuite.robot b/atest/01_SimpleTestSuite.robot index ea8312c..e476823 100755 --- a/atest/01_SimpleTestSuite.robot +++ b/atest/01_SimpleTestSuite.robot @@ -1,48 +1,48 @@ -*** Variables *** -${tc_01} ${0} -${tc_02} ${0} -${retry_1} ${0} -${retry_2} ${0} - - -*** Test Cases *** -My Simple Test - Log Hello World - Log This TRACE message should not be logged! level=TRACE - Should Be Equal Hello Hello - -TC01 - Retry Once - [Tags] test:retry(2) - Log This TRACE message can be logged sometimes level=TRACE - VAR ${tc_01} = ${${tc_01}+1} scope=SUITE - Should Be Equal As Integers ${tc_01} ${2} - -TC01 - Retry Twice - [Tags] test:retry(2) - Log This TRACE message can be logged sometimes level=TRACE - VAR ${tc_02} = ${${tc_02}+1} scope=SUITE - Should Be Equal As Integers ${tc_02} ${3} - -My Simple Test2 - Log Hello World - Log This TRACE message should not be logged! level=TRACE - Should Be Equal Hello Hello - -Log Trace Message - Log This TRACE message should not be logged! level=TRACE - -Passes after 3 Fails - [Tags] test:retry(3) - Log This TRACE message should be logged on failures only! level=TRACE - Should Be Equal ${retry_1} ${3} - [Teardown] Set Suite Variable ${retry_1} ${retry_1 + 1} - -Passes on 4th Exec - [Tags] task:retry(5) - Log This TRACE message should be logged on failures only! level=TRACE - Should Be Equal ${retry_2} ${4} - [Teardown] Set Suite Variable ${retry_2} ${retry_2 + 1} - -My Simple Test1 - Log Hello World - Should Be Equal Hello Hello +*** Variables *** +${tc_01} ${0} +${tc_02} ${0} +${retry_1} ${0} +${retry_2} ${0} + + +*** Test Cases *** +My Simple Test + Log Hello World + Log This TRACE message should not be logged! level=TRACE + Should Be Equal Hello Hello + +TC01 - Retry Once + [Tags] test:retry(2) + Log This TRACE message can be logged sometimes level=TRACE + VAR ${tc_01} = ${${tc_01}+1} scope=SUITE + Should Be Equal As Integers ${tc_01} ${2} + +TC01 - Retry Twice + [Tags] test:retry(2) + Log This TRACE message can be logged sometimes level=TRACE + VAR ${tc_02} = ${${tc_02}+1} scope=SUITE + Should Be Equal As Integers ${tc_02} ${3} + +My Simple Test2 + Log Hello World + Log This TRACE message should not be logged! level=TRACE + Should Be Equal Hello Hello + +Log Trace Message + Log This TRACE message should not be logged! level=TRACE + +Passes after 3 Fails + [Tags] test:retry(3) + Log This TRACE message should be logged on failures only! level=TRACE + Should Be Equal ${retry_1} ${3} + [Teardown] Set Suite Variable ${retry_1} ${retry_1 + 1} + +Passes on 4th Exec + [Tags] task:retry(5) + Log This TRACE message should be logged on failures only! level=TRACE + Should Be Equal ${retry_2} ${4} + [Teardown] Set Suite Variable ${retry_2} ${retry_2 + 1} + +My Simple Test1 + Log Hello World + Should Be Equal Hello Hello diff --git a/atest/02_KeywordRetryListener.robot b/atest/02_KeywordRetryListener.robot index 8590363..58f8f28 100644 --- a/atest/02_KeywordRetryListener.robot +++ b/atest/02_KeywordRetryListener.robot @@ -21,7 +21,7 @@ Test 3 - Failed Child Keyword - Passed on Multiple Retries Test 4 - Failed - Retry Test & Keywords [Tags] test:retry(10) - + Test 4 - Parent Keyword Test 5 - Failed Multiple Child Keywords - Pass on Retry @@ -36,8 +36,8 @@ Successful Keyword Log Successful Error - Pass on Retry - [Arguments] ${successful_retry: int} [Tags] keyword:retry(2) + [Arguments] ${successful_retry: int} ${counter_01} = Evaluate $counter_01 + 1 VAR ${counter_01} = ${counter_01} scope=SUITE Should Be Equal ${counter_01} ${successful_retry} @@ -53,8 +53,8 @@ Test 4 - Parent Keyword Test 4 - Child Keyword 20 Test 4 - Child Keyword - [Arguments] ${successful_retry: int} [Tags] keyword:retry(2) + [Arguments] ${successful_retry: int} ${counter_02} = Evaluate $counter_02 + 1 VAR ${counter_02} = ${counter_02} scope=GLOBAL Should Be Equal ${counter_02} ${successful_retry} @@ -70,12 +70,8 @@ Test 5 - First Child Keyword Test 5 - Second Child Keyword 70 Test 5 - Second Child Keyword - [Arguments] ${successful_retry: int} [Tags] keyword:retry(2) + [Arguments] ${successful_retry: int} ${counter_03} = Evaluate $counter_03 + 1 VAR ${counter_03} = ${counter_03} scope=GLOBAL Should Be Equal ${counter_03} ${successful_retry} - - - - diff --git a/atest/03_KeywordRetry.robot b/atest/03_KeywordRetry.robot new file mode 100644 index 0000000..a546581 --- /dev/null +++ b/atest/03_KeywordRetry.robot @@ -0,0 +1,66 @@ +*** Settings *** +Library KeywordRetry.py + + +*** Test Cases *** +Test 0 - Low Level Retry PASS + [Tags] pass + Retry Three Times 3 test + +Test 1 - Low Level Retry PASS with Setup and Teardown + [Tags] pass + [Setup] Retry Three Times 3 setup + Retry Three Times 3 test + [Teardown] Retry Three Times 3 teardown + +Test 2 - Low Level Retry FAIL in Setup + [Tags] fail + [Setup] Retry Three Times 5 setup + Retry Three Times 5 test + [Teardown] Retry Three Times 5 teardown + +Test 3 - Low Level Retry FAIL in Test + [Tags] fail + Retry Three Times 5 test + +Test 4 - User level Retry PASS + [Tags] pass + VAR ${retries} ${1} scope=TEST + User Level Retry Three Times 3 retries + +Test 5 - User level Retry FAIL + [Tags] fail + VAR ${retries} ${1} scope=TEST + User Level Retry Three Times 5 retries + +Test 6 - User level Retry PASS with Teardown + [Tags] fail + VAR ${keyword} ${1} scope=TEST + VAR ${teardown} ${1} scope=TEST + User Level Retry Three Times 3 keyword + [Teardown] User Level Retry Three Times 3 teardown + +Test 7 - User level Retry FAIL with Teardown + [Tags] fail + VAR ${keyword} ${1} scope=TEST + VAR ${teardown} ${1} scope=TEST + User Level Retry Three Times 5 keyword + [Teardown] User Level Retry Three Times 5 teardown + + +*** Keywords *** +High Level Pass + Log High Level Pass + +User Level With Low level Retry + [Arguments] ${attempts} + Retry Three Times ${attempts} + +User Level Retry Three Times + [Tags] keyword:retry(3) + [Arguments] ${attempts: int} ${variable: str}=retries + TRY + Should Be True $${variable} == $attempts + FINALLY + Inc Test Variable By Name ${variable} + END diff --git a/atest/KeywordRetry.py b/atest/KeywordRetry.py new file mode 100644 index 0000000..9daa87f --- /dev/null +++ b/atest/KeywordRetry.py @@ -0,0 +1,26 @@ +from robot.api import logger +from robot.api.deco import keyword +from robot.libraries.BuiltIn import BuiltIn + + +class KeywordRetry: + def __init__(self) -> None: + self.retries: dict[str, int] = {} + + @keyword(tags=["keyword:retry(3)"]) + def retry_three_times(self, attempts: int = 3, alias: str = "Retry Three Times") -> None: + """A keyword that is set to be retried 3 times upon failure.""" + try: + assert self.retries.get(alias, 1) == attempts, ( + f"Attempt {self.retries.get(alias, 0)} failed." + ) + except AssertionError as e: + self.retries[alias] = self.retries.get(alias, 1) + 1 + raise e + + @keyword + def inc_test_variable_by_name(self, name: str) -> None: + """Sets a variable in the Robot Framework context by its name.""" + value = BuiltIn().get_variable_value(f"${{{name}}}", 0) + 1 + logger.info(f"Incrementing variable '${{{name}}}' to {value}") + BuiltIn().set_test_variable(f"${{{name}}}", value) diff --git a/atest/run_atest.sh b/atest/run_atest.sh index 42e06b1..0f0edd0 100755 --- a/atest/run_atest.sh +++ b/atest/run_atest.sh @@ -1 +1 @@ -robot -d results --listener RetryFailed:10:True:TRACE 01_SimpleTestSuite.robot \ No newline at end of file +robot -d results --listener RetryFailed --loglevel TRACE . \ No newline at end of file diff --git a/bootstrap.py b/bootstrap.py new file mode 100644 index 0000000..20ae0bd --- /dev/null +++ b/bootstrap.py @@ -0,0 +1,71 @@ +import platform +import subprocess +from pathlib import Path +from venv import EnvBuilder + + +class Colors: + """ANSI color codes""" + + BLACK = "\033[0;30m" + RED = "\033[0;31m" + GREEN = "\033[0;32m" + BROWN = "\033[0;33m" + BLUE = "\033[0;34m" + PURPLE = "\033[0;35m" + CYAN = "\033[0;36m" + LIGHT_GRAY = "\033[0;37m" + DARK_GRAY = "\033[1;30m" + LIGHT_RED = "\033[1;31m" + LIGHT_GREEN = "\033[1;32m" + YELLOW = "\033[1;33m" + LIGHT_BLUE = "\033[1;34m" + LIGHT_PURPLE = "\033[1;35m" + LIGHT_CYAN = "\033[1;36m" + LIGHT_WHITE = "\033[1;37m" + BOLD = "\033[1m" + FAINT = "\033[2m" + ITALIC = "\033[3m" + UNDERLINE = "\033[4m" + BLINK = "\033[5m" + NEGATIVE = "\033[7m" + CROSSED = "\033[9m" + END = "\033[0m" + # cancel SGR codes if we don't write to a terminal + if not __import__("sys").stdout.isatty(): + for _ in dir(): + if isinstance(_, str) and _[0] != "_": + locals()[_] = "" + elif __import__("platform").system() == "Windows": + kernel32 = __import__("ctypes").windll.kernel32 + kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7) + del kernel32 + + +venv_dir = Path() / ".venv" +if not platform.platform().startswith("Windows"): + venv_bin = venv_dir / "bin" + venv_python = venv_bin / "python" + venv_pre_commit = venv_bin / "pre-commit" +else: + venv_bin = venv_dir / "Scripts" + venv_python = venv_bin / "python.exe" + venv_pre_commit = venv_bin / "pre-commit.exe" + +if not venv_dir.exists(): + print(f"Creating virtualenv in {venv_dir}") + EnvBuilder(with_pip=True).create(venv_dir) + +subprocess.run([venv_python, "-m", "pip", "install", "-U", "pip"], check=False) +subprocess.run([venv_python, "-m", "pip", "install", "-e", ".[dev]"], check=False) +subprocess.run([venv_python, "-m", "pip", "install", "-e", ".[test]"], check=False) +subprocess.run([venv_pre_commit, "install"], check=False) + +activate_script = ( + "source .venv/bin/activate" + if not platform.platform().startswith("Windows") + else ".venv\\Scripts\\activate" +) + +print(f"\n\nVirtualenv '{Colors.GREEN}{venv_dir}{Colors.END}' is ready and up-to-date.") +print(f"Run '{Colors.GREEN}{activate_script}{Colors.END}' to activate the virtualenv.") diff --git a/createPip_whl_tar.sh b/createPip_whl_tar.sh deleted file mode 100755 index d9dd7a8..0000000 --- a/createPip_whl_tar.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/zsh -check-manifest --update -rm -f dist/*.* -python setup.py bdist_wheel sdist -twine check dist/* -echo "Next step - uploading to pypi!" -read -n 1 -twine upload dist/* diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..cf40d4c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,112 @@ +[build-system] +requires = ["flit_core>=3.11,<4"] +build-backend = "flit_core.buildapi" + +[project] +name = "robotframework-retryfailed" +description = "Robot Framework listener that automatically retries tagged tests, tasks, or keywords." +readme = "README.md" +license = { file = "LICENSE" } +requires-python = ">=3.10" +authors = [ + { name = "René Rohner", email = "snooz@posteo.de" }, +] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Topic :: Software Development :: Testing", + "Topic :: Software Development :: Testing :: Acceptance", + "Framework :: Robot Framework", +] +dependencies = [ + "robotframework>=7.0,<8.0", +] +dynamic = ["version"] + +[project.optional-dependencies] +dev = [ + "check-manifest>=0.50", + "flit>=3.12,<4", + "mypy>=1.15.0", + "pre-commit>=4.1.0", + "ruff>=0.9.6", + "twine>=5.1.0", + "robotframework-robocop>=6.7.0", +] +test = [ + "robotframework-robocop>=6.7.0", +] + +[project.urls] +Source = "https://github.com/MarketSquare/robotframework-retryfailed" +Issues = "https://github.com/MarketSquare/robotframework-retryfailed/issues" + +[tool.flit.module] +name = "RetryFailed" + +[tool.ruff] +line-length = 100 +target-version = "py310" +src = ["src"] +exclude = [ + ".venv", + ".git", + "__pycache__", + "build", + "dist", + "bootstrap.py", + "atest/", +] + +[tool.ruff.lint] +select = [ + "E", + "F", + "W", + "C90", + "I", + "N", + "B", + "PYI", + "PL", + "PTH", + "UP", + "A", + "C4", + "DTZ", + "ISC", + "ICN", + "INP", + "PIE", + "T20", + "PYI", + "PT", + "RSE", + "RET", + "SIM", + "RUF" +] +ignore = [ + "N999", # Robot library names stay CamelCase +] +unfixable = [] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" + +[tool.mypy] +python_version = "3.10" +strict = true +show_error_codes = true +warn_return_any = true +warn_unused_configs = true +warn_unreachable = true +files = ["src"] +exclude = "/(\\.venv|\\.git|__pycache__|build|dist|atest)/" +disable_error_code = ["import-untyped"] diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 2a2de1e..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,3 +0,0 @@ -robotframework>=4.1.0 -setuptools -twine \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 80468ba..0000000 --- a/setup.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Copyright 2022- René Rohner - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License.""" - -import re -from pathlib import Path - -from setuptools import find_packages, setup - -with open("README.md", "r", encoding="utf-8") as fh: - long_description = fh.read() - -with open(Path("src", "RetryFailed", "__init__.py"), encoding="utf-8") as f: - VERSION = re.search('\n__version__ = "(.*)"', f.read()).group(1) - -setup( - name="robotframework-retryfailed", - version=VERSION, - author="René Rohner(Snooz82)", - author_email="snooz@posteo.de", - description="A listener to automatically retry tests or tasks based on flags.", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/MarketSquare/robotframework-retryfailed", - package_dir={"": "src"}, - packages=find_packages("src"), - classifiers=[ - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - "Topic :: Software Development :: Testing", - "Topic :: Software Development :: Testing :: Acceptance", - "Framework :: Robot Framework", - ], - install_requires=["robotframework >= 4.1"], - python_requires=">=3.8.0", -) diff --git a/src/RetryFailed/__init__.py b/src/RetryFailed/__init__.py index 945e2e2..9912654 100644 --- a/src/RetryFailed/__init__.py +++ b/src/RetryFailed/__init__.py @@ -14,4 +14,6 @@ from .retry_failed import RetryFailed +__all__ = ["RetryFailed"] + __version__ = "0.2.0" diff --git a/src/RetryFailed/retry_failed.py b/src/RetryFailed/retry_failed.py index 53f83cf..92e137b 100644 --- a/src/RetryFailed/retry_failed.py +++ b/src/RetryFailed/retry_failed.py @@ -14,26 +14,34 @@ import copy import re - from dataclasses import dataclass from pathlib import Path +from uuid import uuid4 -from robot.api import ExecutionResult, ResultVisitor from robot.api.deco import library +from robot.api.interfaces import ListenerV3 from robot.libraries.BuiltIn import BuiltIn -from robot.utils.robottypes import is_truthy +from robot.model import Error as ModelError +from robot.model import TestSuite as ModelTestSuite +from robot.result import ExecutionResult, ResultVisitor from robot.result import Keyword as ResultKeyword +from robot.result import Message as ResultMessage +from robot.result import TestCase as ResultTestCase +from robot.result import TestSuite as ResultTestSuite from robot.running import Keyword as RunningKeyword from robot.running import TestCase as RunningTestCase -from robot.result import TestCase as ResultTestCase +from robot.running import TestSuite as RunningTestSuite +from robot.utils.robottypes import is_truthy duplicate_test_pattern = re.compile( r"Multiple .*? with name '(?P.*?)' executed in.*? suite '(?P.*?)'." ) linebreak = "\n" + @dataclass class KeywordMetaData: + kw_uuid: str kw_obj: RunningKeyword kw_index: int kw_name: str @@ -43,19 +51,16 @@ class KeywordMetaData: retries_performed: int - @library(scope="GLOBAL") -class RetryFailed: - - ROBOT_LISTENER_API_VERSION = 3 - - def __init__(self, - global_test_retries: int = 0, - keep_retried_tests: bool = False, - log_level: str | None = None, - warn_on_test_retry: bool = True, - warn_on_kw_retry: bool = False - ): +class RetryFailed(ListenerV3): + def __init__( + self, + global_test_retries: int = 0, + keep_retried_tests: bool = False, + log_level: str | None = None, + warn_on_test_retry: bool = True, + warn_on_kw_retry: bool = False, + ): self.ROBOT_LIBRARY_LISTENER = self # Generic Settings @@ -63,13 +68,14 @@ def __init__(self, self.warn_on_kw_retry: bool = is_truthy(warn_on_kw_retry) # TestRetryListener - self.retried_tests = [] + self.retried_tests: list[str] = [] self.retries = 0 self._max_retries_by_default = int(global_test_retries) self.max_retries = global_test_retries self.keep_retried_tests = is_truthy(keep_retried_tests) self.log_level = log_level - self._original_log_level = None + self.kw_control_log_level: str | None = None + self._original_log_level: str | None = None self.test_retry_active: bool = False self.original_testcase_object: RunningTestCase = None @@ -77,7 +83,7 @@ def __init__(self, self.retry_keywords: list[KeywordMetaData] = [] self._index_counter: int = 1 - def start_test(self, test: RunningTestCase, result: ResultTestCase): + def start_test(self, test: RunningTestCase, result: ResultTestCase) -> None: if self.retries: BuiltIn().set_test_variable("${RETRYFAILED_RETRY_INDEX}", self.retries) if self.log_level is not None: @@ -92,133 +98,137 @@ def start_test(self, test: RunningTestCase, result: ResultTestCase): self.max_retries = self._max_retries_by_default return - def end_test(self, test: RunningTestCase, result: ResultTestCase): - if self.retries and self._original_log_level is not None: - BuiltIn().set_log_level(self._original_log_level) - if not self.max_retries: - self.retries = 0 - return - if result.status == "FAIL": - if self.retries < self.max_retries: - self.test_retry_active = True - index = test.parent.tests.index(test) - test.parent.tests.insert(index + 1, copy.deepcopy(self.original_testcase_object)) - result.status = "SKIP" - result.message += "\nSkipped for Retry" - self.retried_tests.append(test.longname) - self.retries += 1 - return - else: - self.test_retry_active = False - result.message += ( - f"{linebreak * bool(result.message)}[RETRY] FAIL on {self.retries}. retry." - ) - else: - if self.retries: - self.test_retry_active = False - result.message += ( - f"{linebreak * bool(result.message)}[RETRY] PASS on {self.retries}. retry." - ) - self.retries = 0 - return - - def end_suite(self, suite, result): - test_dict = {} - result_dict = {} - for result_test, test in zip(result.tests, suite.tests): - test_dict[test.id] = test - result_dict[test.id] = result_test - result.tests = list(result_dict.values()) - suite.tests = list(test_dict.values()) - - def start_keyword(self, keyword: RunningKeyword, result: ResultKeyword): - + def start_keyword(self, keyword: RunningKeyword, result: ResultKeyword) -> None: for tag in result.tags: retry_kw = re.match(r"keyword:retry\((\d+)\)", tag) if not retry_kw: return _retries = int(retry_kw.group(1)) if retry_kw and _retries > 0: - kw_data = KeywordMetaData ( - kw_obj = keyword, - kw_index = keyword.parent.body.index(keyword), - kw_name = keyword.name, - kw_source = Path(keyword.source).name, - kw_lineno = keyword.lineno, - retries = _retries, - retries_performed = 0, + kw_data = KeywordMetaData( + kw_uuid=str(uuid4), + kw_obj=keyword, + kw_index=keyword.parent.body.index(keyword), + kw_name=keyword.name, + kw_source=Path(keyword.source).name, + kw_lineno=keyword.lineno, + retries=_retries, + retries_performed=0, ) # check if keyword is already registered for the retry for registered_retry_keyword in self.retry_keywords: if ( - registered_retry_keyword.kw_name == kw_data.kw_name - and - registered_retry_keyword.kw_source == kw_data.kw_source - and - registered_retry_keyword.kw_lineno == kw_data.kw_lineno - ): + registered_retry_keyword.kw_name == kw_data.kw_name + and registered_retry_keyword.kw_source == kw_data.kw_source + and registered_retry_keyword.kw_lineno == kw_data.kw_lineno + ): return self.retry_keywords.append(kw_data) - def end_keyword(self, keyword: RunningKeyword, result: ResultKeyword): - - executed_kw_name = keyword.name - executed_kw_source = Path(keyword.source).name - - # reset original loglevel - if self._original_log_level: - BuiltIn().set_log_level(self._original_log_level) - + def end_keyword(self, keyword: RunningKeyword, result: ResultKeyword) -> None: match_kw_retry = False kw_to_retry: KeywordMetaData for index, kw in enumerate(self.retry_keywords): - if kw.kw_name != executed_kw_name or kw.kw_source != executed_kw_source: + if kw.kw_name != keyword.name or kw.kw_source != Path(keyword.source).name: continue - else: - match_kw_retry = True - current_index = index - kw_to_retry = kw + match_kw_retry = True + current_index = index + kw_to_retry = kw # If currently executed keyword does not match any defined RetryKeyword -> just return if not match_kw_retry: return - + link = self._get_keyword_link(result) + level: str = "WARN" if self.warn_on_kw_retry else "INFO" if result.status == "PASS": - doc = f"[Keyword: {kw_to_retry.kw_name}] PASSED on {kw_to_retry.retries_performed}. retry." - msg = f"[Keyword: {link}] PASSED on {kw_to_retry.retries_performed}. retry." + # reset log level + self.reset_log_level(kw_to_retry) + + # log specific message if keyword PASSED on retry if kw_to_retry.retries_performed > 0: - level: str = "WARN" if self.warn_on_kw_retry else "INFO" + doc = ( + f"[Keyword: {kw_to_retry.kw_name}] PASSED on " + f"{kw_to_retry.retries_performed}. retry." + ) + msg = f"[Keyword: {link}] PASSED on {kw_to_retry.retries_performed}. retry." BuiltIn().log(msg, level=level, html=True) result.doc += f"\n\n{doc}" self.retry_keywords.pop(current_index) if result.status == "FAIL": - if kw_to_retry.retries and kw_to_retry.retries_performed < kw_to_retry.retries: # Set loglevel for retry if self.log_level: + self.kw_control_log_level = kw_to_retry.kw_uuid self._original_log_level = BuiltIn().set_log_level(self.log_level) - keyword.parent.body.insert(kw_to_retry.kw_index + self._index_counter, kw_to_retry.kw_obj) + keyword.parent.body.insert( + kw_to_retry.kw_index + self._index_counter, kw_to_retry.kw_obj + ) result.status = "NOT RUN" kw_to_retry.retries_performed += 1 self.retry_keywords[current_index].retries_performed = kw_to_retry.retries_performed - msg = f"[Keyword: {kw_to_retry.kw_name}] - Skipped for {self.retry_keywords[current_index].retries_performed}. Retry..." + performed = self.retry_keywords[current_index].retries_performed + msg = f"[Keyword: {kw_to_retry.kw_name}] - Skipped for {performed}. Retry..." result.doc += f"\n\n{msg}" else: - doc = f"{"\n\n" * bool(result.message)}[Keyword: {kw_to_retry.kw_name}] FAILED on {kw_to_retry.retries_performed}. retry." - msg = f"{"\n\n" * bool(result.message)}[Keyword: {link}] FAILED on {self.retry_keywords[current_index].retries_performed}. retry." + # reset log level + self.reset_log_level(kw_to_retry) + + prefix = "\n\n" if result.message else "" + doc = ( + f"{prefix}[Keyword: {kw_to_retry.kw_name}] FAILED on " + f"{kw_to_retry.retries_performed}. retry." + ) + performed = self.retry_keywords[current_index].retries_performed + msg = f"{prefix}[Keyword: {link}] FAILED on {performed}. retry." result.doc += doc result.message += doc - level: str = "WARN" if self.warn_on_kw_retry else "INFO" BuiltIn().log(msg.replace("\n", ""), level=level, html=True) self.retry_keywords.pop(current_index) - def message(self, message): + def end_test(self, test: RunningTestCase, result: ResultTestCase) -> None: + if self.retries and self._original_log_level is not None: + BuiltIn().set_log_level(self._original_log_level) + if not self.max_retries: + self.retries = 0 + return + if result.status == "FAIL": + if self.retries < self.max_retries: + self.test_retry_active = True + index = test.parent.tests.index(test) + test.parent.tests.insert(index + 1, copy.deepcopy(self.original_testcase_object)) + result.status = "SKIP" + result.message += "\nSkipped for Retry" + self.retried_tests.append(test.longname) + self.retries += 1 + return + self.test_retry_active = False + result.message += ( + f"{linebreak * bool(result.message)}[RETRY] FAIL on {self.retries}. retry." + ) + elif self.retries: + self.test_retry_active = False + result.message += ( + f"{linebreak * bool(result.message)}[RETRY] PASS on {self.retries}. retry." + ) + self.retries = 0 + return + + def end_suite(self, suite: RunningTestSuite, result: ResultTestSuite) -> None: + test_dict = {} + result_dict = {} + for result_test, test in zip(result.tests, suite.tests, strict=False): + test_dict[test.id] = test + result_dict[test.id] = result_test + result.tests = list(result_dict.values()) + suite.tests = list(test_dict.values()) + + def message(self, message: ResultMessage) -> None: if message.level == "WARN": match = duplicate_test_pattern.match(message.message) if match and f"{match.group('suite')}.{match.group('test')}" in self.retried_tests: @@ -228,13 +238,17 @@ def message(self, message): if not self.warn_on_test_retry: message.level = "INFO" - def output_file(self, original_output_xml): + def output_file(self, original_output_xml: Path | None) -> None: + if original_output_xml is None: + return result = ExecutionResult(original_output_xml) - result.visit(RetryMerger(self.retried_tests, self.keep_retried_tests, self.warn_on_test_retry)) + result.visit( + RetryMerger(self.retried_tests, self.keep_retried_tests, self.warn_on_test_retry) + ) result.save() - - def _get_keyword_link(self, keyword_result: ResultKeyword): - link = ( + + def _get_keyword_link(self, keyword_result: ResultKeyword) -> str: + return ( f" None: + """ + Reset to original loglevel if keyword uuid does match to the keyword + which has initially modified the loglevel. + """ + if self._original_log_level and kw_object.kw_uuid == self.kw_control_log_level: + self.kw_control_log_level = None + self._original_log_level = None + BuiltIn().set_log_level(self._original_log_level) -class RetryMerger(ResultVisitor): - def __init__(self, retried_tests, keep_retried_tests=False, warn_on_test_retry: bool = True): +class RetryMerger(ResultVisitor): # type: ignore[misc] + def __init__( + self, + retried_tests: list[str], + keep_retried_tests: bool = False, + warn_on_test_retry: bool = True, + ): self.retried_tests = retried_tests self.keep_retried_tests = keep_retried_tests self.warn_on_test_retry = warn_on_test_retry - self.test_ids = {} + self.test_ids: dict[str, str] = {} - def start_suite(self, suite): + def start_suite(self, suite: ModelTestSuite) -> None: if self.keep_retried_tests: return test_dict = {} @@ -262,12 +290,12 @@ def start_suite(self, suite): test_dict[test.name] = test suite.tests = list(test_dict.values()) - def end_suite(self, suite): + def end_suite(self, suite: ModelTestSuite) -> None: for test in suite.tests: if test.longname in self.retried_tests: self.test_ids[test.name] = test.id - def start_errors(self, errors): + def start_errors(self, errors: ModelError) -> None: messages = [] retry_messages = {} for message in errors.messages: @@ -290,9 +318,9 @@ def start_errors(self, errors): messages + list(retry_messages.values()), key=lambda m: m.timestamp ) - def _get_test_link(self, test_name): + def _get_test_link(self, test_name: str) -> str: test_id = self.test_ids.get(test_name) - link = ( + return ( f" Date: Mon, 2 Mar 2026 17:02:17 +0100 Subject: [PATCH 4/8] apply new strategy for keyword retries (#4) * added state for ongoing kw retry * strategy for discussions * new strategy - draft * fix retry count mechanism * retry handling + log handling * warn instead error --------- Co-authored-by: Marvin Klerx Co-authored-by: Marvin Klerx --- STRATEGY.md | 25 +++ atest/02_KeywordRetryListener.robot | 6 + atest/03_KeywordRetry.robot | 44 ++--- src/RetryFailed/retry_failed.py | 240 +++++++++++++--------------- 4 files changed, 157 insertions(+), 158 deletions(-) create mode 100644 STRATEGY.md diff --git a/STRATEGY.md b/STRATEGY.md new file mode 100644 index 0000000..523c5d8 --- /dev/null +++ b/STRATEGY.md @@ -0,0 +1,25 @@ +# Keyword Retry Listener + +## KW in Test Case / Keyword Body + +Workflow: + +- start_keyword: if kw has retry tag -> register kw in internal list with metadata +- end_keyword: if kw failed & kw registered for retry -> append kw to parent body object (test / keyword) + +--> keyword will be retried automatically as it is the next item in the parent test / keyword body. + +## KW in Setup / Teardown (Suite / Test) + +Issues: + +- Suite setup: parent object of executed & failed keyword doesnt have a body object. +- Generic setup / teardown issue: those types are not part of the test / kw body - retry keyword cannot be appended to list + +Solutions: +- suite setup: ??? +- suite teardown: ??? +- test setup: do we want to retry the complete test ? +- test teardown: ??? + +Or do we want to call the ``run_keyword`` function from the listener to directly retry the failed keywords ? If yes, must be different handling than keywords called directly from the test / kw body... \ No newline at end of file diff --git a/atest/02_KeywordRetryListener.robot b/atest/02_KeywordRetryListener.robot index 58f8f28..8acf435 100644 --- a/atest/02_KeywordRetryListener.robot +++ b/atest/02_KeywordRetryListener.robot @@ -29,6 +29,12 @@ Test 5 - Failed Multiple Child Keywords - Pass on Retry Test 5 - Parent Keyword +Test 6 - Direct Keyword Call in Setup / Teardown - Expected Failure + [Setup] Error - Pass on Retry 5 + [Teardown] Run Keyword If $TEST_STATUS == "PASS" Fail Test was expected to fail but it did pass! + [Tags] robot:skip-on-failure + Log Test should fail! + *** Keywords *** Successful Keyword diff --git a/atest/03_KeywordRetry.robot b/atest/03_KeywordRetry.robot index a546581..741a0c3 100644 --- a/atest/03_KeywordRetry.robot +++ b/atest/03_KeywordRetry.robot @@ -3,24 +3,13 @@ Library KeywordRetry.py *** Test Cases *** -Test 0 - Low Level Retry PASS +Test 1 - Low Level Retry PASS [Tags] pass Retry Three Times 3 test -Test 1 - Low Level Retry PASS with Setup and Teardown - [Tags] pass - [Setup] Retry Three Times 3 setup - Retry Three Times 3 test - [Teardown] Retry Three Times 3 teardown - -Test 2 - Low Level Retry FAIL in Setup - [Tags] fail - [Setup] Retry Three Times 5 setup - Retry Three Times 5 test - [Teardown] Retry Three Times 5 teardown - -Test 3 - Low Level Retry FAIL in Test - [Tags] fail +Test 2 - Low Level Retry FAIL in Test + [Tags] fail robot:skip-on-failure + [Teardown] Run Keyword If $TEST_STATUS == "PASS" Fail Test was expected to fail but it did pass! Retry Three Times 5 test Test 4 - User level Retry PASS @@ -29,26 +18,23 @@ Test 4 - User level Retry PASS User Level Retry Three Times 3 retries Test 5 - User level Retry FAIL - [Tags] fail + [Tags] fail robot:skip-on-failure + [Teardown] Run Keyword If $TEST_STATUS == "PASS" Fail Test was expected to fail but it did pass! VAR ${retries} ${1} scope=TEST User Level Retry Three Times 5 retries -Test 6 - User level Retry PASS with Teardown - [Tags] fail - VAR ${keyword} ${1} scope=TEST - VAR ${teardown} ${1} scope=TEST - User Level Retry Three Times 3 keyword - [Teardown] User Level Retry Three Times 3 teardown - -Test 7 - User level Retry FAIL with Teardown - [Tags] fail - VAR ${keyword} ${1} scope=TEST - VAR ${teardown} ${1} scope=TEST - User Level Retry Three Times 5 keyword - [Teardown] User Level Retry Three Times 5 teardown +Test 6 - Recurse Keyword Retry + Recurse 0 10 *** Keywords *** +Recurse + [Tags] keyword:retry(4) + [Arguments] ${arg: int} ${pass_on_count: int}=3 + IF $arg == $pass_on_count RETURN + + Recurse ${arg + 1} ${pass_on_count} + High Level Pass Log High Level Pass diff --git a/src/RetryFailed/retry_failed.py b/src/RetryFailed/retry_failed.py index 92e137b..c68e5d2 100644 --- a/src/RetryFailed/retry_failed.py +++ b/src/RetryFailed/retry_failed.py @@ -16,10 +16,12 @@ import re from dataclasses import dataclass from pathlib import Path -from uuid import uuid4 +from typing import Any, Literal +from robot.api import logger from robot.api.deco import library from robot.api.interfaces import ListenerV3 +from robot.api.logger import LogLevel from robot.libraries.BuiltIn import BuiltIn from robot.model import Error as ModelError from robot.model import TestSuite as ModelTestSuite @@ -40,15 +42,9 @@ @dataclass -class KeywordMetaData: - kw_uuid: str - kw_obj: RunningKeyword - kw_index: int - kw_name: str - kw_source: str - kw_lineno: int - retries: int - retries_performed: int +class RetryKeyword: + keyword: RunningKeyword + remaining_retries: int @library(scope="GLOBAL") @@ -57,7 +53,7 @@ def __init__( self, global_test_retries: int = 0, keep_retried_tests: bool = False, - log_level: str | None = None, + log_level: LogLevel | None = None, warn_on_test_retry: bool = True, warn_on_kw_retry: bool = False, ): @@ -69,154 +65,116 @@ def __init__( # TestRetryListener self.retried_tests: list[str] = [] - self.retries = 0 + self.test_retries = 0 self._max_retries_by_default = int(global_test_retries) self.max_retries = global_test_retries self.keep_retried_tests = is_truthy(keep_retried_tests) - self.log_level = log_level - self.kw_control_log_level: str | None = None - self._original_log_level: str | None = None + self.log_level: LogLevel | None = log_level + self.initial_log_level: str | None = None self.test_retry_active: bool = False self.original_testcase_object: RunningTestCase = None # KeywordRetryListener - self.retry_keywords: list[KeywordMetaData] = [] - self._index_counter: int = 1 - - def start_test(self, test: RunningTestCase, result: ResultTestCase) -> None: - if self.retries: - BuiltIn().set_test_variable("${RETRYFAILED_RETRY_INDEX}", self.retries) - if self.log_level is not None: - self._original_log_level = BuiltIn().set_log_level(self.log_level) - if self.retries == 0 and not self.test_retry_active: + self.retry_stack: list[RetryKeyword] = [] + + # Regex to identify retry definition in tags + self.kw_retry_regex = r"keyword:retry\((\d+)\)" + self.test_retry_regex = r"(?:test|task):retry\((\d+)\)" + + def start_test(self, test: RunningTestCase, _: ResultTestCase) -> None: + if self.test_retries: + BuiltIn().set_test_variable("${RETRYFAILED_RETRY_INDEX}", self.test_retries) + if self.test_retries == 0 and not self.test_retry_active: self.original_testcase_object = copy.deepcopy(test) - for tag in test.tags: - retry_match = re.match(r"(?:test|task):retry\((\d+)\)", tag) - if retry_match: - self.max_retries = int(retry_match.group(1)) - return + + retries = self._check_if_retry(test.tags, "TEST") + if retries: + self.max_retries = retries + return self.max_retries = self._max_retries_by_default return - def start_keyword(self, keyword: RunningKeyword, result: ResultKeyword) -> None: - for tag in result.tags: - retry_kw = re.match(r"keyword:retry\((\d+)\)", tag) - if not retry_kw: - return - _retries = int(retry_kw.group(1)) - if retry_kw and _retries > 0: - kw_data = KeywordMetaData( - kw_uuid=str(uuid4), - kw_obj=keyword, - kw_index=keyword.parent.body.index(keyword), - kw_name=keyword.name, - kw_source=Path(keyword.source).name, - kw_lineno=keyword.lineno, - retries=_retries, - retries_performed=0, - ) - - # check if keyword is already registered for the retry - for registered_retry_keyword in self.retry_keywords: - if ( - registered_retry_keyword.kw_name == kw_data.kw_name - and registered_retry_keyword.kw_source == kw_data.kw_source - and registered_retry_keyword.kw_lineno == kw_data.kw_lineno - ): - return - self.retry_keywords.append(kw_data) - - def end_keyword(self, keyword: RunningKeyword, result: ResultKeyword) -> None: - match_kw_retry = False - kw_to_retry: KeywordMetaData - for index, kw in enumerate(self.retry_keywords): - if kw.kw_name != keyword.name or kw.kw_source != Path(keyword.source).name: - continue - match_kw_retry = True - current_index = index - kw_to_retry = kw + def end_keyword(self, keyword: RunningKeyword, result: ResultKeyword) -> Any: - # If currently executed keyword does not match any defined RetryKeyword -> just return - if not match_kw_retry: + # if keyword is not registered for retries -> return + if not (retries := self._check_if_retry(result.tags, "KEYWORD")): return - link = self._get_keyword_link(result) - level: str = "WARN" if self.warn_on_kw_retry else "INFO" - - if result.status == "PASS": - # reset log level - self.reset_log_level(kw_to_retry) + level: LogLevel = "WARN" if self.warn_on_kw_retry else "INFO" - # log specific message if keyword PASSED on retry - if kw_to_retry.retries_performed > 0: - doc = ( - f"[Keyword: {kw_to_retry.kw_name}] PASSED on " - f"{kw_to_retry.retries_performed}. retry." - ) - msg = f"[Keyword: {link}] PASSED on {kw_to_retry.retries_performed}. retry." + if result.status != "FAIL": + if self.retry_stack and self.retry_stack[-1].keyword == keyword: + doc = f"[Keyword: {keyword.name}] PASSED on {retries - self.retry_stack[-1].remaining_retries}. retry." # noqa + msg = f"[Keyword: {self._get_keyword_link(result)}] PASSED on {retries - self.retry_stack[-1].remaining_retries}. retry." # noqa BuiltIn().log(msg, level=level, html=True) result.doc += f"\n\n{doc}" - self.retry_keywords.pop(current_index) - - if result.status == "FAIL": - if kw_to_retry.retries and kw_to_retry.retries_performed < kw_to_retry.retries: - # Set loglevel for retry - if self.log_level: - self.kw_control_log_level = kw_to_retry.kw_uuid - self._original_log_level = BuiltIn().set_log_level(self.log_level) + self.retry_stack.pop() + if not self.retry_stack and not self.test_retry_active: + self.reset_loglevel() + return - keyword.parent.body.insert( - kw_to_retry.kw_index + self._index_counter, kw_to_retry.kw_obj - ) - result.status = "NOT RUN" - kw_to_retry.retries_performed += 1 - self.retry_keywords[current_index].retries_performed = kw_to_retry.retries_performed + if keyword.type in ("SETUP", "TEARDOWN"): + BuiltIn().log( + "Keyword in SETUP & TEARDOWN can't be retried directly - use wrapper keyword!", + level="WARN", + html=True, + ) + return - performed = self.retry_keywords[current_index].retries_performed - msg = f"[Keyword: {kw_to_retry.kw_name}] - Skipped for {performed}. Retry..." + # keyword is already getting retried + if self.retry_stack and self.retry_stack[-1].keyword == keyword: + # all retries have been executed and keyword still failed + if not self.retry_stack[-1].remaining_retries: + msg = f"Keyword '{keyword.name}' FAILED after {retries - self.retry_stack[-1].remaining_retries}. retry!" # noqa + self.retry_stack.pop() result.doc += f"\n\n{msg}" - else: - # reset log level - self.reset_log_level(kw_to_retry) - - prefix = "\n\n" if result.message else "" - doc = ( - f"{prefix}[Keyword: {kw_to_retry.kw_name}] FAILED on " - f"{kw_to_retry.retries_performed}. retry." - ) - performed = self.retry_keywords[current_index].retries_performed - msg = f"{prefix}[Keyword: {link}] FAILED on {performed}. retry." - result.doc += doc - result.message += doc - BuiltIn().log(msg.replace("\n", ""), level=level, html=True) - self.retry_keywords.pop(current_index) + BuiltIn().log(msg, level=level, html=True) + self.reset_loglevel() + return + # keyword failure gets detected the first time + else: + self.retry_stack.append(RetryKeyword(keyword, retries)) + + msg = f"Keyword '{keyword.name}' - Perform {retries - self.retry_stack[-1].remaining_retries + 1}. retry..." # noqa + BuiltIn().log(msg, level=level, html=True) + + # insert keyword to the next executing index in the parent object + result.status = "NOT RUN" + keyword.parent.body.insert(keyword.parent.body.index(keyword), keyword) + self.retry_stack[-1].remaining_retries -= 1 + + # set log level in case of keyword must be retried + if self.log_level: + self.set_loglevel(self.log_level) def end_test(self, test: RunningTestCase, result: ResultTestCase) -> None: - if self.retries and self._original_log_level is not None: - BuiltIn().set_log_level(self._original_log_level) if not self.max_retries: - self.retries = 0 + self.test_retries = 0 return if result.status == "FAIL": - if self.retries < self.max_retries: + if self.test_retries < self.max_retries: + if self.log_level: + self.set_loglevel(self.log_level) self.test_retry_active = True index = test.parent.tests.index(test) test.parent.tests.insert(index + 1, copy.deepcopy(self.original_testcase_object)) result.status = "SKIP" result.message += "\nSkipped for Retry" self.retried_tests.append(test.longname) - self.retries += 1 + self.test_retries += 1 return self.test_retry_active = False result.message += ( - f"{linebreak * bool(result.message)}[RETRY] FAIL on {self.retries}. retry." + f"{linebreak * bool(result.message)}[RETRY] FAIL on {self.test_retries}. retry." ) - elif self.retries: + elif self.test_retries: self.test_retry_active = False result.message += ( - f"{linebreak * bool(result.message)}[RETRY] PASS on {self.retries}. retry." + f"{linebreak * bool(result.message)}[RETRY] PASS on {self.test_retries}. retry." ) - self.retries = 0 + if self.log_level: + self.reset_loglevel() + self.test_retries = 0 return def end_suite(self, suite: RunningTestSuite, result: ResultTestSuite) -> None: @@ -233,7 +191,7 @@ def message(self, message: ResultMessage) -> None: match = duplicate_test_pattern.match(message.message) if match and f"{match.group('suite')}.{match.group('test')}" in self.retried_tests: message.message = ( - f"Retry {self.retries}/{self.max_retries} of test '{match.group('test')}':" + f"Retry {self.test_retries}/{self.max_retries} of test '{match.group('test')}':" ) if not self.warn_on_test_retry: message.level = "INFO" @@ -259,15 +217,39 @@ def _get_keyword_link(self, keyword_result: ResultKeyword) -> str: else keyword_result.kwname ) - def reset_log_level(self, kw_object: KeywordMetaData) -> None: + def _check_if_retry(self, tags: list[str], token: Literal["TEST", "KEYWORD"]) -> int: + """ + Function checks if the given test / keyword should be retried or not - defined by their tags + """ + for tag in tags: + regex = self.kw_retry_regex if token == "KEYWORD" else self.test_retry_regex + retry_kw = re.match(regex, tag) + if not retry_kw: + continue + return int(retry_kw.group(1)) + return 0 + + def set_loglevel( + self, + level: LogLevel | None, + ) -> None: + """ + Custom function to set robot log level correctly. + """ + if BuiltIn()._context.output.log_level.level == self.log_level: + return + self.initial_log_level = BuiltIn()._context.output.set_log_level(level) + BuiltIn()._namespace.variables.set_global("${LOG_LEVEL}", level) + if BuiltIn()._context.output.log_level.level != self.log_level: + logger.warn("Setting log level failed!") + + def reset_loglevel(self) -> None: """ - Reset to original loglevel if keyword uuid does match to the keyword - which has initially modified the loglevel. + Custom function to reset robot log level correctly. """ - if self._original_log_level and kw_object.kw_uuid == self.kw_control_log_level: - self.kw_control_log_level = None - self._original_log_level = None - BuiltIn().set_log_level(self._original_log_level) + BuiltIn().reset_log_level() + if BuiltIn()._context.output.log_level.level != self.initial_log_level: + logger.warn("Resetting log level failed!") class RetryMerger(ResultVisitor): # type: ignore[misc] From 48c218f4cc0e2c4f78bd53a1d18c55ac38667922 Mon Sep 17 00:00:00 2001 From: Marvin Klerx Date: Mon, 2 Mar 2026 17:04:12 +0100 Subject: [PATCH 5/8] remove file --- STRATEGY.md | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 STRATEGY.md diff --git a/STRATEGY.md b/STRATEGY.md deleted file mode 100644 index 523c5d8..0000000 --- a/STRATEGY.md +++ /dev/null @@ -1,25 +0,0 @@ -# Keyword Retry Listener - -## KW in Test Case / Keyword Body - -Workflow: - -- start_keyword: if kw has retry tag -> register kw in internal list with metadata -- end_keyword: if kw failed & kw registered for retry -> append kw to parent body object (test / keyword) - ---> keyword will be retried automatically as it is the next item in the parent test / keyword body. - -## KW in Setup / Teardown (Suite / Test) - -Issues: - -- Suite setup: parent object of executed & failed keyword doesnt have a body object. -- Generic setup / teardown issue: those types are not part of the test / kw body - retry keyword cannot be appended to list - -Solutions: -- suite setup: ??? -- suite teardown: ??? -- test setup: do we want to retry the complete test ? -- test teardown: ??? - -Or do we want to call the ``run_keyword`` function from the listener to directly retry the failed keywords ? If yes, must be different handling than keywords called directly from the test / kw body... \ No newline at end of file From 34a83b639da9e774f3c4813d2defd31c5a0052f6 Mon Sep 17 00:00:00 2001 From: Marvin Klerx Date: Mon, 2 Mar 2026 17:10:53 +0100 Subject: [PATCH 6/8] fix loglevel setter --- src/RetryFailed/retry_failed.py | 37 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/src/RetryFailed/retry_failed.py b/src/RetryFailed/retry_failed.py index c68e5d2..b62b719 100644 --- a/src/RetryFailed/retry_failed.py +++ b/src/RetryFailed/retry_failed.py @@ -29,10 +29,8 @@ from robot.result import Keyword as ResultKeyword from robot.result import Message as ResultMessage from robot.result import TestCase as ResultTestCase -from robot.result import TestSuite as ResultTestSuite from robot.running import Keyword as RunningKeyword from robot.running import TestCase as RunningTestCase -from robot.running import TestSuite as RunningTestSuite from robot.utils.robottypes import is_truthy duplicate_test_pattern = re.compile( @@ -145,7 +143,7 @@ def end_keyword(self, keyword: RunningKeyword, result: ResultKeyword) -> Any: # set log level in case of keyword must be retried if self.log_level: - self.set_loglevel(self.log_level) + self.set_loglevel() def end_test(self, test: RunningTestCase, result: ResultTestCase) -> None: if not self.max_retries: @@ -154,7 +152,7 @@ def end_test(self, test: RunningTestCase, result: ResultTestCase) -> None: if result.status == "FAIL": if self.test_retries < self.max_retries: if self.log_level: - self.set_loglevel(self.log_level) + self.set_loglevel() self.test_retry_active = True index = test.parent.tests.index(test) test.parent.tests.insert(index + 1, copy.deepcopy(self.original_testcase_object)) @@ -177,14 +175,14 @@ def end_test(self, test: RunningTestCase, result: ResultTestCase) -> None: self.test_retries = 0 return - def end_suite(self, suite: RunningTestSuite, result: ResultTestSuite) -> None: - test_dict = {} - result_dict = {} - for result_test, test in zip(result.tests, suite.tests, strict=False): - test_dict[test.id] = test - result_dict[test.id] = result_test - result.tests = list(result_dict.values()) - suite.tests = list(test_dict.values()) + # def end_suite(self, suite: RunningTestSuite, result: ResultTestSuite) -> None: + # test_dict = {} + # result_dict = {} + # for result_test, test in zip(result.tests, suite.tests, strict=False): + # test_dict[test.id] = test + # result_dict[test.id] = result_test + # result.tests = list(result_dict.values()) + # suite.tests = list(test_dict.values()) def message(self, message: ResultMessage) -> None: if message.level == "WARN": @@ -229,17 +227,14 @@ def _check_if_retry(self, tags: list[str], token: Literal["TEST", "KEYWORD"]) -> return int(retry_kw.group(1)) return 0 - def set_loglevel( - self, - level: LogLevel | None, - ) -> None: + def set_loglevel(self) -> None: """ Custom function to set robot log level correctly. """ if BuiltIn()._context.output.log_level.level == self.log_level: return - self.initial_log_level = BuiltIn()._context.output.set_log_level(level) - BuiltIn()._namespace.variables.set_global("${LOG_LEVEL}", level) + self.initial_log_level = BuiltIn()._context.output.set_log_level(self.log_level) + BuiltIn()._namespace.variables.set_global("${LOG_LEVEL}", self.log_level) if BuiltIn()._context.output.log_level.level != self.log_level: logger.warn("Setting log level failed!") @@ -247,7 +242,11 @@ def reset_loglevel(self) -> None: """ Custom function to reset robot log level correctly. """ - BuiltIn().reset_log_level() + _initial_log_level = BuiltIn()._context.output.initial_log_level + if BuiltIn()._context.output.log_level.level == _initial_log_level: + return + BuiltIn()._context.output.set_log_level(_initial_log_level) + BuiltIn()._namespace.variables.set_global("${LOG_LEVEL}", _initial_log_level) if BuiltIn()._context.output.log_level.level != self.initial_log_level: logger.warn("Resetting log level failed!") From bd352d2371d6a2d5c6f5aa48f385263737b401cf Mon Sep 17 00:00:00 2001 From: Marvin Klerx Date: Mon, 2 Mar 2026 17:12:48 +0100 Subject: [PATCH 7/8] removed end suite function --- src/RetryFailed/retry_failed.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/RetryFailed/retry_failed.py b/src/RetryFailed/retry_failed.py index b62b719..8561b2a 100644 --- a/src/RetryFailed/retry_failed.py +++ b/src/RetryFailed/retry_failed.py @@ -175,15 +175,6 @@ def end_test(self, test: RunningTestCase, result: ResultTestCase) -> None: self.test_retries = 0 return - # def end_suite(self, suite: RunningTestSuite, result: ResultTestSuite) -> None: - # test_dict = {} - # result_dict = {} - # for result_test, test in zip(result.tests, suite.tests, strict=False): - # test_dict[test.id] = test - # result_dict[test.id] = result_test - # result.tests = list(result_dict.values()) - # suite.tests = list(test_dict.values()) - def message(self, message: ResultMessage) -> None: if message.level == "WARN": match = duplicate_test_pattern.match(message.message) From a2a55aead03f626259ccb5cd0f619b34a5508933 Mon Sep 17 00:00:00 2001 From: Marvin Klerx Date: Mon, 2 Mar 2026 17:25:21 +0100 Subject: [PATCH 8/8] added atests to pre-commit config --- .pre-commit-config.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 652e285..972e6d7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,3 +28,14 @@ repos: language: python types_or: [python, pyi, jupyter] require_serial: true + + # Robot framework acceptance tests + - repo: local + hooks: + - id: acceptance-tests + name: acceptance-tests + description: "Run Robot Framework acceptance tests" + entry: robot -d results --listener RetryFailed:0:True:TRACE --loglevel INFO atest + language: system + pass_filenames: false + always_run: true