diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..d9ad27c --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,34 @@ +# robotframework-retryfailed – Copilot Guide + +## Architecture +- `src/RetryFailed/retry_failed.py` hosts the entire listener/library; `RetryFailed` exposes ROBOT_LISTENER_API_VERSION 3 and is imported via `RetryFailed` package (see `src/RetryFailed/__init__.py`). +- `KeywordMetaData` snapshots keyword name/source/line so we can reinsert the same `RunningKeyword` when retrying without duplicating registrations. +- `start_test` deep-copies the original `RunningTestCase` and sets `${RETRYFAILED_RETRY_INDEX}`; `end_test` requeues failures by inserting the saved test back into `test.parent.tests` and flagging original results as SKIP. +- `end_suite` and `RetryMerger` (a `robot.api.ResultVisitor`) rewrite the execution tree so persisted output either collapses duplicates or keeps them when `keep_retried_tests=True`. + +## Retry semantics +- Tests/tasks opt in via tags like `[Tags] test:retry(2)` or `task:retry(5)`; missing tags fall back to the constructor’s `global_test_retries` default supplied on the `--listener RetryFailed:` CLI. +- Keyword retries are entirely tag-driven: `[Tags] keyword:retry()` on either user keywords or test cases, with `start_keyword` storing metadata and `end_keyword` ensuring only one registration per (name, source, lineno). +- The listener exposes two warning toggles (`warn_on_test_retry`, `warn_on_kw_retry`) that gate whether BuiltIn logs use WARN or INFO when retries happen; preserve these semantics whenever adding new log messages. +- When `log_level` is passed, retries temporarily raise Robot’s log level; always reset through `_original_log_level` before exiting the listener hook to avoid leaking state between tests/keywords. + +## Result shaping & messaging +- `message()` intercepts duplicate-test WARNs emitted by Robot and rewrites them into retry status lines so final logs stay meaningful; any new warning flow should piggyback on this formatting pattern (`Retry {x}/{y} of test ...`). +- `output_file()` always rewrites `output.xml` in place via `ExecutionResult.visit(RetryMerger(...))`; if you touch result data structures remember that post-processing happens after Robot finishes. +- HTML links inserted by `_get_keyword_link()` and `_get_test_link()` rely on Robot-generated element ids; keep IDs intact when adjusting result objects. + +## Tests & verification +- Acceptance tests live under `atest/`; run both suites locally with `robot -d results --listener RetryFailed:10:True:TRACE atest/01_SimpleTestSuite.robot` and `atest/02_KeywordRetryListener.robot` (scripted in `atest/run_atest.sh`). +- The suites rely on stateful suite variables (`VAR … scope=SUITE/GLOBAL`), so run them serially and reset `${counter_*}` variables when writing new cases. +- There are no unit tests; regression coverage hinges on these Robot suites plus manual log inspection of `output.xml`/`log.html`. + +## Development workflow +- Use Python ≥3.8; `pip install -r requirements-dev.txt` installs the project in editable mode plus the `dev` extra (flit, mypy, ruff, robocop, check-manifest, twine). +- Packaging now runs through Flit: bump `__version__` in `src/RetryFailed/__init__.py`, then execute `./createPip_whl_tar.sh` (wraps `flit build`, `twine check`, and `flit publish`). +- `.pre-commit-config.yaml` is authoritative (ruff lint/format, mypy, robocop); the same settings live in `pyproject.toml`, so add new tooling there. + +## Contribution conventions +- Prefer pure Python stdlib + Robot APIs; new deps must be justified because the listener is imported inside Robot runs. +- Keep listener hooks fast and side-effect free—no network/file IO inside `start_*`/`end_*` without caching since they run per test/keyword. +- Document any new listener args in both the README table and `atest` tags so behavior stays discoverable. +- When adding retry metadata, key lookups currently use `(kw_name, kw_source, kw_lineno)`; preserve or extend that tuple instead of relying solely on names to avoid clashes between identically named keywords in different files. diff --git a/.gitignore b/.gitignore index ea1a214..5622e95 100644 --- a/.gitignore +++ b/.gitignore @@ -129,3 +129,8 @@ dmypy.json .pyre/ atest/results results +log.html +output.xml +report.html +.vscode/launch.json +temp_listener.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..972e6d7 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,41 @@ +repos: + # mypy linter for Python files + - repo: local + hooks: + - id: mypy + name: mypy + description: 'Run mypy for static type checking' + entry: mypy + language: python + types_or: [python, pyi] + require_serial: true + + # Ruff linter for Python files + - repo: local + hooks: + - id: ruff-format + name: ruff-format + description: "Run 'ruff format' for extremely fast Python formatting" + entry: ruff format --force-exclude + language: python + types_or: [python, pyi, jupyter] + require_serial: true + + - id: ruff + name: ruff + description: "Run 'ruff' for extremely fast Python linting" + entry: ruff check --force-exclude + language: python + types_or: [python, pyi, jupyter] + require_serial: true + + # Robot framework acceptance tests + - repo: local + hooks: + - id: acceptance-tests + name: acceptance-tests + description: "Run Robot Framework acceptance tests" + entry: robot -d results --listener RetryFailed:0:True:TRACE --loglevel INFO atest + language: system + pass_filenames: false + always_run: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..c9198b0 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,58 @@ +# Contributing to robotframework-retryfailed + +Thanks for helping improve the RetryFailed listener! This project is intentionally small, so the +workflow is simple and scripted. The steps below explain how to set up an environment, run +tooling, and publish a release. + +## 1. Environment setup +1. Use Python 3.8+. +2. Create a virtual environment and install the dev extras: + ```bash + python -m venv .venv + source .venv/bin/activate + pip install -r requirements-dev.txt + ``` + The `dev` extra installs flit, mypy, ruff, robocop, check-manifest, twine, and Robot Framework + add-ons used in the acceptance suites. +3. Install the pre-commit hooks (optional but recommended): + ```bash + pre-commit install + ``` + +## 2. Linting & formatting +- **Python formatting**: `ruff format .` +- **Python linting**: `ruff check .` +- **Static typing**: `mypy src` +- **Robot Framework linting**: `robocop check --include *.robot --include *.resource` +- **Robot Framework formatting**: `robocop format --include *.robot --include *.resource` +- **Pre-commit (full suite)**: `pre-commit run --all-files` + +The same settings live in `pyproject.toml`, so CI and local runs stay aligned. Prefer running the +standalone commands when iterating quickly, and fall back to the pre-commit aggregate before +delivering work. + +## 3. Tests +All regression coverage is provided by the acceptance suites under `atest/`. Run them from the +repository root with the listener installed in editable mode: +```bash +robot -d results --listener RetryFailed:10:True:TRACE atest/01_SimpleTestSuite.robot +robot -d results --listener RetryFailed:10:True:TRACE atest/02_KeywordRetryListener.robot +``` +Inspect `results/output.xml` and `results/log.html` to confirm retries are recorded correctly. + +## 4. Release workflow +1. Bump `__version__` inside `src/RetryFailed/__init__.py`. +2. Verify the manifest and build artifacts: + ```bash + check-manifest --update + flit build + twine check dist/* + ``` +3. Upload to PyPI (after any manual smoke tests): + ```bash + flit publish + ``` + The helper script `./createPip_whl_tar.sh` automates the same sequence with a keypress between + verification and upload. + +Please open an issue before large or breaking changes so we can align on direction. Happy hacking! diff --git a/README.md b/README.md index 0f91b9d..1778b35 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,25 @@ # robotframework-retryfailed -A listener to automatically retry tests or tasks based on tags. +A listener to automatically retry tests, tasks or keywords based on tags. ## Installation Install with pip: +``` +pip install robotframework-retryfailed +``` - pip install robotframework-retryfailed +## CLI Arguments + +You can configure the following CLI arguments when registering the listener in your robotframework cli call: + +| Argument | Description | Mandatory | Default Value | +|----------|-------------|-----------|---------------| +| ``global_test_retries`` | Define a global number of retries which is valid for ALL your tests by default! | No | `**0** | +| ``keep_retried_tests`` | Define if the retried tests should be kept in the logs or not. If ``True``, they will be marked with status ``Skip`` | No | **False** | +| ``log_level`` | If set, the loglevel will be changed to the given value IF a test / keyword is getting retried. | No | **None** | +| ``warn_on_test_retry`` | If ``True``, the retried tests will be logged as warning to the ``log.html`` | No | **True** | +| ``warn_on_kw_retry`` | If ``True``, the retried keywords will be logged as warning to the ``log.html`` | No | **False** | ## Usage @@ -14,27 +27,54 @@ Add the listener to your robot execution, via command line arguments. When your tests do fail and you have tagged them with `test:retry(2)`, it will retry the test 2 times. Retry can be also set globally as a parameter to the listener. -### Attaching Listener +### Attaching Listener - Retry Tests Example: +``` +# Attaching listener without any default retry configuratio +robot --listener RetryFailed - robot --listener RetryFailed +# Attach listener & retrying every failed tests once if failed +robot --listener RetryFailed:1 - robot --listener RetryFailed:1 +# Attaching listener, retrying every tests once, keep failed tests in logfile & increase loglevel to TRACE for retry tests. +robot --listener RetryFailed:1:True:TRACE +``` -Second one will by default retry once every failing test. +### Retry Test Cases - Tagging Tests -### Tagging Tests - -Example: +If attaching the listener without any default retry configuration, you must set the count of max. retry as ``Test Tag``. - *** Test Cases *** - Test Case - [Tags] test:retry(2) - Log This test will be retried 2 times if it fails +See Example: +``` +*** Test Cases *** +Test Case + [Tags] test:retry(2) + Log This test will be retried 2 times if it fails +``` Tagging tasks by `task:retry(3)` should also work. +### Retry Keywords - Tagging Keywords + +The ``KeywordRetryListener`` is basically always ``activated`` & needs to be defined by yourself within your test. + +It makes no sense to configure a default retry count for every keyword, because usually too many keywords are part a test / parent keyword to retry them **ALL** ! +This means, you need to define the keywords which should be retried by yourself! + +Therefore, you must configure the following ``Keyword Tags``: +``` +*** Keywords *** +KeywordABC + [Documentation] Keyword takes max. 1 retry! + [Tags] keyword:retry(1) + +KeywordDEF + [Documentation] Keyword takes max. 5 retries! + [Tags] keyword:retry(5) +``` + + ### Configuration On top of specifying the number of retries, you can also define whether your want to **keep the logs** of the retried tests and change the **log level** when retrying, by providing respectfully second and third parameter to the listener: `RetryFailed:::` @@ -42,12 +82,28 @@ On top of specifying the number of retries, you can also define whether your wan By default the logs of the retried tests are not kept, and the log level remains the same as the regular run. Example: +``` +# keep the logs of the retried tests +robot --listener RetryFailed:1:True + +# does not keep the logs of the retried tests and change log level to DEBUG when retrying +robot --listener RetryFailed:2:False:DEBUG + +# keep the logs of the retried tests and change the log level to TRACE when retrying +robot --listener RetryFailed:1:True:TRACE + +# Same like previous one, but keep in mind: all retried tests are getting logged as warning. But all retried keywords are not getting logged as warning! +robot --listener RetryFailed:1:True:TRACE + +# Both retried tests & retried keywords are getting logged as warning into the log.html +robot --listener RetryFailed:1:True:TRACE:True:True + +# Only retried keywords are getting logged as warning into the log.html +robot --listener RetryFailed:1:True:TRACE:False:True +``` - # keep the logs of the retried tests - robot --listener RetryFailed:1:True +### Log Warnings at Retry - # does not keep the logs of the retried tests and change log level to DEBUG when retrying - robot --listener RetryFailed:2:False:DEBUG +If you've set both parameters, ``warn_on_test_retry`` & ``warn_on_kw_retry``, to ``False``, a simple ``Info`` message gets logged during the keyword execution in the log.html. - # keep the logs of the retried tests and change the log level to TRACE when retrying - robot --listener RetryFailed:1:True:TRACE +You won't see at the top of the log file, but you can find it directly within the logged keyword execution. \ No newline at end of file diff --git a/atest/01_SimpleTestSuite.robot b/atest/01_SimpleTestSuite.robot index 1f1a194..e476823 100755 --- a/atest/01_SimpleTestSuite.robot +++ b/atest/01_SimpleTestSuite.robot @@ -1,39 +1,48 @@ -*** Settings *** -# Library RetryFailed log_level=TRACE - - -*** Variables *** -${retry_1} ${0} -${retry_2} ${0} - - -*** Test Cases *** -My Simple Test - Log Hello World - Should Be Equal Hello Hello - -Sometime Fail - [Tags] test:retry(1) - Should Be True ${{random.randint(0, 1)}} == 1 - -Sometime Fail1 - [Tags] test:retry(3) - Should Be True ${{random.randint(0, 1)}} == 1 - -Sometime Fail2 - [Tags] test:retry(1) - Should Be True ${{random.randint(0, 1)}} == 1 - -Passes after 3 Fails - [Tags] test:retry(3) - Should Be Equal ${retry_1} ${3} - [Teardown] Set Suite Variable ${retry_1} ${retry_1 + 1} - -Fails on 4th Exec - [Tags] task:retry(5) - Should Be Equal ${retry_2} ${4} - [Teardown] Set Suite Variable ${retry_2} ${retry_2 + 1} - -My Simple Test1 - Log Hello World - Should Be Equal Hello Hello +*** Variables *** +${tc_01} ${0} +${tc_02} ${0} +${retry_1} ${0} +${retry_2} ${0} + + +*** Test Cases *** +My Simple Test + Log Hello World + Log This TRACE message should not be logged! level=TRACE + Should Be Equal Hello Hello + +TC01 - Retry Once + [Tags] test:retry(2) + Log This TRACE message can be logged sometimes level=TRACE + VAR ${tc_01} = ${${tc_01}+1} scope=SUITE + Should Be Equal As Integers ${tc_01} ${2} + +TC01 - Retry Twice + [Tags] test:retry(2) + Log This TRACE message can be logged sometimes level=TRACE + VAR ${tc_02} = ${${tc_02}+1} scope=SUITE + Should Be Equal As Integers ${tc_02} ${3} + +My Simple Test2 + Log Hello World + Log This TRACE message should not be logged! level=TRACE + Should Be Equal Hello Hello + +Log Trace Message + Log This TRACE message should not be logged! level=TRACE + +Passes after 3 Fails + [Tags] test:retry(3) + Log This TRACE message should be logged on failures only! level=TRACE + Should Be Equal ${retry_1} ${3} + [Teardown] Set Suite Variable ${retry_1} ${retry_1 + 1} + +Passes on 4th Exec + [Tags] task:retry(5) + Log This TRACE message should be logged on failures only! level=TRACE + Should Be Equal ${retry_2} ${4} + [Teardown] Set Suite Variable ${retry_2} ${retry_2 + 1} + +My Simple Test1 + Log Hello World + Should Be Equal Hello Hello diff --git a/atest/02_KeywordRetryListener.robot b/atest/02_KeywordRetryListener.robot new file mode 100644 index 0000000..8acf435 --- /dev/null +++ b/atest/02_KeywordRetryListener.robot @@ -0,0 +1,83 @@ +*** Variables *** +${counter_01: int} = 0 +${counter_02: int} = 0 +${counter_03: int} = 0 + + +*** Test Cases *** +Test 1 - PASS + [Tags] test:retry(0) + Successful Keyword + +Test 2 - Failed - Pass on 2. Retry + [Tags] test:retry(0) + Error - Pass on Retry 2 + +Test 3 - Failed Child Keyword - Passed on Multiple Retries + [Tags] test:retry(0) + VAR ${counter_01} = ${0} scope=suite + + Test 3 - Failed Child Keyword + +Test 4 - Failed - Retry Test & Keywords + [Tags] test:retry(10) + + Test 4 - Parent Keyword + +Test 5 - Failed Multiple Child Keywords - Pass on Retry + [Tags] test:retry(2) + + Test 5 - Parent Keyword + +Test 6 - Direct Keyword Call in Setup / Teardown - Expected Failure + [Setup] Error - Pass on Retry 5 + [Teardown] Run Keyword If $TEST_STATUS == "PASS" Fail Test was expected to fail but it did pass! + [Tags] robot:skip-on-failure + Log Test should fail! + + +*** Keywords *** +Successful Keyword + [Tags] keyword:retry(2) + Log Successful + +Error - Pass on Retry + [Tags] keyword:retry(2) + [Arguments] ${successful_retry: int} + ${counter_01} = Evaluate $counter_01 + 1 + VAR ${counter_01} = ${counter_01} scope=SUITE + Should Be Equal ${counter_01} ${successful_retry} + +Test 3 - Failed Child Keyword + [Tags] keyword:retry(3) + + Error - Pass on Retry 5 + +Test 4 - Parent Keyword + [Tags] keyword:retry(2) + + Test 4 - Child Keyword 20 + +Test 4 - Child Keyword + [Tags] keyword:retry(2) + [Arguments] ${successful_retry: int} + ${counter_02} = Evaluate $counter_02 + 1 + VAR ${counter_02} = ${counter_02} scope=GLOBAL + Should Be Equal ${counter_02} ${successful_retry} + +Test 5 - Parent Keyword + [Tags] keyword:retry(2) + + Test 5 - First Child Keyword + +Test 5 - First Child Keyword + [Tags] keyword:retry(2) + + Test 5 - Second Child Keyword 70 + +Test 5 - Second Child Keyword + [Tags] keyword:retry(2) + [Arguments] ${successful_retry: int} + ${counter_03} = Evaluate $counter_03 + 1 + VAR ${counter_03} = ${counter_03} scope=GLOBAL + Should Be Equal ${counter_03} ${successful_retry} diff --git a/atest/03_KeywordRetry.robot b/atest/03_KeywordRetry.robot new file mode 100644 index 0000000..741a0c3 --- /dev/null +++ b/atest/03_KeywordRetry.robot @@ -0,0 +1,52 @@ +*** Settings *** +Library KeywordRetry.py + + +*** Test Cases *** +Test 1 - Low Level Retry PASS + [Tags] pass + Retry Three Times 3 test + +Test 2 - Low Level Retry FAIL in Test + [Tags] fail robot:skip-on-failure + [Teardown] Run Keyword If $TEST_STATUS == "PASS" Fail Test was expected to fail but it did pass! + Retry Three Times 5 test + +Test 4 - User level Retry PASS + [Tags] pass + VAR ${retries} ${1} scope=TEST + User Level Retry Three Times 3 retries + +Test 5 - User level Retry FAIL + [Tags] fail robot:skip-on-failure + [Teardown] Run Keyword If $TEST_STATUS == "PASS" Fail Test was expected to fail but it did pass! + VAR ${retries} ${1} scope=TEST + User Level Retry Three Times 5 retries + +Test 6 - Recurse Keyword Retry + Recurse 0 10 + + +*** Keywords *** +Recurse + [Tags] keyword:retry(4) + [Arguments] ${arg: int} ${pass_on_count: int}=3 + IF $arg == $pass_on_count RETURN + + Recurse ${arg + 1} ${pass_on_count} + +High Level Pass + Log High Level Pass + +User Level With Low level Retry + [Arguments] ${attempts} + Retry Three Times ${attempts} + +User Level Retry Three Times + [Tags] keyword:retry(3) + [Arguments] ${attempts: int} ${variable: str}=retries + TRY + Should Be True $${variable} == $attempts + FINALLY + Inc Test Variable By Name ${variable} + END diff --git a/atest/KeywordRetry.py b/atest/KeywordRetry.py new file mode 100644 index 0000000..9daa87f --- /dev/null +++ b/atest/KeywordRetry.py @@ -0,0 +1,26 @@ +from robot.api import logger +from robot.api.deco import keyword +from robot.libraries.BuiltIn import BuiltIn + + +class KeywordRetry: + def __init__(self) -> None: + self.retries: dict[str, int] = {} + + @keyword(tags=["keyword:retry(3)"]) + def retry_three_times(self, attempts: int = 3, alias: str = "Retry Three Times") -> None: + """A keyword that is set to be retried 3 times upon failure.""" + try: + assert self.retries.get(alias, 1) == attempts, ( + f"Attempt {self.retries.get(alias, 0)} failed." + ) + except AssertionError as e: + self.retries[alias] = self.retries.get(alias, 1) + 1 + raise e + + @keyword + def inc_test_variable_by_name(self, name: str) -> None: + """Sets a variable in the Robot Framework context by its name.""" + value = BuiltIn().get_variable_value(f"${{{name}}}", 0) + 1 + logger.info(f"Incrementing variable '${{{name}}}' to {value}") + BuiltIn().set_test_variable(f"${{{name}}}", value) diff --git a/atest/run_atest.sh b/atest/run_atest.sh index 171757c..0f0edd0 100755 --- a/atest/run_atest.sh +++ b/atest/run_atest.sh @@ -1 +1 @@ -robot -d results --listener RetryFailed 01_SimpleTestSuite.robot \ No newline at end of file +robot -d results --listener RetryFailed --loglevel TRACE . \ No newline at end of file diff --git a/bootstrap.py b/bootstrap.py new file mode 100644 index 0000000..20ae0bd --- /dev/null +++ b/bootstrap.py @@ -0,0 +1,71 @@ +import platform +import subprocess +from pathlib import Path +from venv import EnvBuilder + + +class Colors: + """ANSI color codes""" + + BLACK = "\033[0;30m" + RED = "\033[0;31m" + GREEN = "\033[0;32m" + BROWN = "\033[0;33m" + BLUE = "\033[0;34m" + PURPLE = "\033[0;35m" + CYAN = "\033[0;36m" + LIGHT_GRAY = "\033[0;37m" + DARK_GRAY = "\033[1;30m" + LIGHT_RED = "\033[1;31m" + LIGHT_GREEN = "\033[1;32m" + YELLOW = "\033[1;33m" + LIGHT_BLUE = "\033[1;34m" + LIGHT_PURPLE = "\033[1;35m" + LIGHT_CYAN = "\033[1;36m" + LIGHT_WHITE = "\033[1;37m" + BOLD = "\033[1m" + FAINT = "\033[2m" + ITALIC = "\033[3m" + UNDERLINE = "\033[4m" + BLINK = "\033[5m" + NEGATIVE = "\033[7m" + CROSSED = "\033[9m" + END = "\033[0m" + # cancel SGR codes if we don't write to a terminal + if not __import__("sys").stdout.isatty(): + for _ in dir(): + if isinstance(_, str) and _[0] != "_": + locals()[_] = "" + elif __import__("platform").system() == "Windows": + kernel32 = __import__("ctypes").windll.kernel32 + kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7) + del kernel32 + + +venv_dir = Path() / ".venv" +if not platform.platform().startswith("Windows"): + venv_bin = venv_dir / "bin" + venv_python = venv_bin / "python" + venv_pre_commit = venv_bin / "pre-commit" +else: + venv_bin = venv_dir / "Scripts" + venv_python = venv_bin / "python.exe" + venv_pre_commit = venv_bin / "pre-commit.exe" + +if not venv_dir.exists(): + print(f"Creating virtualenv in {venv_dir}") + EnvBuilder(with_pip=True).create(venv_dir) + +subprocess.run([venv_python, "-m", "pip", "install", "-U", "pip"], check=False) +subprocess.run([venv_python, "-m", "pip", "install", "-e", ".[dev]"], check=False) +subprocess.run([venv_python, "-m", "pip", "install", "-e", ".[test]"], check=False) +subprocess.run([venv_pre_commit, "install"], check=False) + +activate_script = ( + "source .venv/bin/activate" + if not platform.platform().startswith("Windows") + else ".venv\\Scripts\\activate" +) + +print(f"\n\nVirtualenv '{Colors.GREEN}{venv_dir}{Colors.END}' is ready and up-to-date.") +print(f"Run '{Colors.GREEN}{activate_script}{Colors.END}' to activate the virtualenv.") diff --git a/createPip_whl_tar.sh b/createPip_whl_tar.sh deleted file mode 100755 index d9dd7a8..0000000 --- a/createPip_whl_tar.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/zsh -check-manifest --update -rm -f dist/*.* -python setup.py bdist_wheel sdist -twine check dist/* -echo "Next step - uploading to pypi!" -read -n 1 -twine upload dist/* diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..cf40d4c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,112 @@ +[build-system] +requires = ["flit_core>=3.11,<4"] +build-backend = "flit_core.buildapi" + +[project] +name = "robotframework-retryfailed" +description = "Robot Framework listener that automatically retries tagged tests, tasks, or keywords." +readme = "README.md" +license = { file = "LICENSE" } +requires-python = ">=3.10" +authors = [ + { name = "René Rohner", email = "snooz@posteo.de" }, +] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Topic :: Software Development :: Testing", + "Topic :: Software Development :: Testing :: Acceptance", + "Framework :: Robot Framework", +] +dependencies = [ + "robotframework>=7.0,<8.0", +] +dynamic = ["version"] + +[project.optional-dependencies] +dev = [ + "check-manifest>=0.50", + "flit>=3.12,<4", + "mypy>=1.15.0", + "pre-commit>=4.1.0", + "ruff>=0.9.6", + "twine>=5.1.0", + "robotframework-robocop>=6.7.0", +] +test = [ + "robotframework-robocop>=6.7.0", +] + +[project.urls] +Source = "https://github.com/MarketSquare/robotframework-retryfailed" +Issues = "https://github.com/MarketSquare/robotframework-retryfailed/issues" + +[tool.flit.module] +name = "RetryFailed" + +[tool.ruff] +line-length = 100 +target-version = "py310" +src = ["src"] +exclude = [ + ".venv", + ".git", + "__pycache__", + "build", + "dist", + "bootstrap.py", + "atest/", +] + +[tool.ruff.lint] +select = [ + "E", + "F", + "W", + "C90", + "I", + "N", + "B", + "PYI", + "PL", + "PTH", + "UP", + "A", + "C4", + "DTZ", + "ISC", + "ICN", + "INP", + "PIE", + "T20", + "PYI", + "PT", + "RSE", + "RET", + "SIM", + "RUF" +] +ignore = [ + "N999", # Robot library names stay CamelCase +] +unfixable = [] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" + +[tool.mypy] +python_version = "3.10" +strict = true +show_error_codes = true +warn_return_any = true +warn_unused_configs = true +warn_unreachable = true +files = ["src"] +exclude = "/(\\.venv|\\.git|__pycache__|build|dist|atest)/" +disable_error_code = ["import-untyped"] diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 2a2de1e..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,3 +0,0 @@ -robotframework>=4.1.0 -setuptools -twine \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 80468ba..0000000 --- a/setup.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Copyright 2022- René Rohner - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License.""" - -import re -from pathlib import Path - -from setuptools import find_packages, setup - -with open("README.md", "r", encoding="utf-8") as fh: - long_description = fh.read() - -with open(Path("src", "RetryFailed", "__init__.py"), encoding="utf-8") as f: - VERSION = re.search('\n__version__ = "(.*)"', f.read()).group(1) - -setup( - name="robotframework-retryfailed", - version=VERSION, - author="René Rohner(Snooz82)", - author_email="snooz@posteo.de", - description="A listener to automatically retry tests or tasks based on flags.", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/MarketSquare/robotframework-retryfailed", - package_dir={"": "src"}, - packages=find_packages("src"), - classifiers=[ - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - "Topic :: Software Development :: Testing", - "Topic :: Software Development :: Testing :: Acceptance", - "Framework :: Robot Framework", - ], - install_requires=["robotframework >= 4.1"], - python_requires=">=3.8.0", -) diff --git a/src/RetryFailed/__init__.py b/src/RetryFailed/__init__.py index 945e2e2..9912654 100644 --- a/src/RetryFailed/__init__.py +++ b/src/RetryFailed/__init__.py @@ -14,4 +14,6 @@ from .retry_failed import RetryFailed +__all__ = ["RetryFailed"] + __version__ = "0.2.0" diff --git a/src/RetryFailed/retry_failed.py b/src/RetryFailed/retry_failed.py index e31013b..8561b2a 100644 --- a/src/RetryFailed/retry_failed.py +++ b/src/RetryFailed/retry_failed.py @@ -12,11 +12,25 @@ See the License for the specific language governing permissions and limitations under the License.""" +import copy import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal -from robot.api import ExecutionResult, ResultVisitor, logger +from robot.api import logger from robot.api.deco import library +from robot.api.interfaces import ListenerV3 +from robot.api.logger import LogLevel from robot.libraries.BuiltIn import BuiltIn +from robot.model import Error as ModelError +from robot.model import TestSuite as ModelTestSuite +from robot.result import ExecutionResult, ResultVisitor +from robot.result import Keyword as ResultKeyword +from robot.result import Message as ResultMessage +from robot.result import TestCase as ResultTestCase +from robot.running import Keyword as RunningKeyword +from robot.running import TestCase as RunningTestCase from robot.utils.robottypes import is_truthy duplicate_test_pattern = re.compile( @@ -25,91 +39,222 @@ linebreak = "\n" -@library(scope="GLOBAL") -class RetryFailed: +@dataclass +class RetryKeyword: + keyword: RunningKeyword + remaining_retries: int - ROBOT_LISTENER_API_VERSION = 3 - def __init__(self, global_retries=0, keep_retried_tests=False, log_level=None): +@library(scope="GLOBAL") +class RetryFailed(ListenerV3): + def __init__( + self, + global_test_retries: int = 0, + keep_retried_tests: bool = False, + log_level: LogLevel | None = None, + warn_on_test_retry: bool = True, + warn_on_kw_retry: bool = False, + ): self.ROBOT_LIBRARY_LISTENER = self - self.retried_tests = [] - self.retries = 0 - self._max_retries_by_default = int(global_retries) - self.max_retries = global_retries + + # Generic Settings + self.warn_on_test_retry: bool = is_truthy(warn_on_test_retry) + self.warn_on_kw_retry: bool = is_truthy(warn_on_kw_retry) + + # TestRetryListener + self.retried_tests: list[str] = [] + self.test_retries = 0 + self._max_retries_by_default = int(global_test_retries) + self.max_retries = global_test_retries self.keep_retried_tests = is_truthy(keep_retried_tests) - self.log_level = log_level - self._original_log_level = None - - def start_test(self, test, result): - if self.retries: - BuiltIn().set_test_variable("${RETRYFAILED_RETRY_INDEX}", self.retries) - if self.log_level is not None: - self._original_log_level = BuiltIn()._context.output.set_log_level(self.log_level) - for tag in test.tags: - retry_match = re.match(r"(?:test|task):retry\((\d+)\)", tag) - if retry_match: - self.max_retries = int(retry_match.group(1)) - return + self.log_level: LogLevel | None = log_level + self.initial_log_level: str | None = None + self.test_retry_active: bool = False + self.original_testcase_object: RunningTestCase = None + + # KeywordRetryListener + self.retry_stack: list[RetryKeyword] = [] + + # Regex to identify retry definition in tags + self.kw_retry_regex = r"keyword:retry\((\d+)\)" + self.test_retry_regex = r"(?:test|task):retry\((\d+)\)" + + def start_test(self, test: RunningTestCase, _: ResultTestCase) -> None: + if self.test_retries: + BuiltIn().set_test_variable("${RETRYFAILED_RETRY_INDEX}", self.test_retries) + if self.test_retries == 0 and not self.test_retry_active: + self.original_testcase_object = copy.deepcopy(test) + + retries = self._check_if_retry(test.tags, "TEST") + if retries: + self.max_retries = retries + return self.max_retries = self._max_retries_by_default return - def end_test(self, test, result): - if self.retries and self._original_log_level is not None: - BuiltIn()._context.output.set_log_level(self._original_log_level) + def end_keyword(self, keyword: RunningKeyword, result: ResultKeyword) -> Any: + + # if keyword is not registered for retries -> return + if not (retries := self._check_if_retry(result.tags, "KEYWORD")): + return + + level: LogLevel = "WARN" if self.warn_on_kw_retry else "INFO" + + if result.status != "FAIL": + if self.retry_stack and self.retry_stack[-1].keyword == keyword: + doc = f"[Keyword: {keyword.name}] PASSED on {retries - self.retry_stack[-1].remaining_retries}. retry." # noqa + msg = f"[Keyword: {self._get_keyword_link(result)}] PASSED on {retries - self.retry_stack[-1].remaining_retries}. retry." # noqa + BuiltIn().log(msg, level=level, html=True) + result.doc += f"\n\n{doc}" + self.retry_stack.pop() + if not self.retry_stack and not self.test_retry_active: + self.reset_loglevel() + return + + if keyword.type in ("SETUP", "TEARDOWN"): + BuiltIn().log( + "Keyword in SETUP & TEARDOWN can't be retried directly - use wrapper keyword!", + level="WARN", + html=True, + ) + return + + # keyword is already getting retried + if self.retry_stack and self.retry_stack[-1].keyword == keyword: + # all retries have been executed and keyword still failed + if not self.retry_stack[-1].remaining_retries: + msg = f"Keyword '{keyword.name}' FAILED after {retries - self.retry_stack[-1].remaining_retries}. retry!" # noqa + self.retry_stack.pop() + result.doc += f"\n\n{msg}" + BuiltIn().log(msg, level=level, html=True) + self.reset_loglevel() + return + # keyword failure gets detected the first time + else: + self.retry_stack.append(RetryKeyword(keyword, retries)) + + msg = f"Keyword '{keyword.name}' - Perform {retries - self.retry_stack[-1].remaining_retries + 1}. retry..." # noqa + BuiltIn().log(msg, level=level, html=True) + + # insert keyword to the next executing index in the parent object + result.status = "NOT RUN" + keyword.parent.body.insert(keyword.parent.body.index(keyword), keyword) + self.retry_stack[-1].remaining_retries -= 1 + + # set log level in case of keyword must be retried + if self.log_level: + self.set_loglevel() + + def end_test(self, test: RunningTestCase, result: ResultTestCase) -> None: if not self.max_retries: - self.retries = 0 + self.test_retries = 0 return if result.status == "FAIL": - if self.retries < self.max_retries: + if self.test_retries < self.max_retries: + if self.log_level: + self.set_loglevel() + self.test_retry_active = True index = test.parent.tests.index(test) - test.parent.tests.insert(index + 1, test) + test.parent.tests.insert(index + 1, copy.deepcopy(self.original_testcase_object)) result.status = "SKIP" result.message += "\nSkipped for Retry" self.retried_tests.append(test.longname) - self.retries += 1 + self.test_retries += 1 return - else: - result.message += ( - f"{linebreak * bool(result.message)}[RETRY] FAIL on {self.retries}. retry." - ) - else: - if self.retries: - result.message += ( - f"{linebreak * bool(result.message)}[RETRY] PASS on {self.retries}. retry." - ) - self.retries = 0 + self.test_retry_active = False + result.message += ( + f"{linebreak * bool(result.message)}[RETRY] FAIL on {self.test_retries}. retry." + ) + elif self.test_retries: + self.test_retry_active = False + result.message += ( + f"{linebreak * bool(result.message)}[RETRY] PASS on {self.test_retries}. retry." + ) + if self.log_level: + self.reset_loglevel() + self.test_retries = 0 return - def end_suite(self, suite, result): - test_dict = {} - result_dict = {} - for result_test, test in zip(result.tests, suite.tests): - test_dict[test.id] = test - result_dict[test.id] = result_test - result.tests = list(result_dict.values()) - suite.tests = list(test_dict.values()) - - def message(self, message): + def message(self, message: ResultMessage) -> None: if message.level == "WARN": match = duplicate_test_pattern.match(message.message) if match and f"{match.group('suite')}.{match.group('test')}" in self.retried_tests: message.message = ( - f"Retry {self.retries}/{self.max_retries} of test '{match.group('test')}':" + f"Retry {self.test_retries}/{self.max_retries} of test '{match.group('test')}':" ) + if not self.warn_on_test_retry: + message.level = "INFO" - def output_file(self, original_output_xml): + def output_file(self, original_output_xml: Path | None) -> None: + if original_output_xml is None: + return result = ExecutionResult(original_output_xml) - result.visit(RetryMerger(self.retried_tests, self.keep_retried_tests)) + result.visit( + RetryMerger(self.retried_tests, self.keep_retried_tests, self.warn_on_test_retry) + ) result.save() + def _get_keyword_link(self, keyword_result: ResultKeyword) -> str: + return ( + f"' + f"{keyword_result.kwname}" + f"" + if keyword_result.id + else keyword_result.kwname + ) + + def _check_if_retry(self, tags: list[str], token: Literal["TEST", "KEYWORD"]) -> int: + """ + Function checks if the given test / keyword should be retried or not - defined by their tags + """ + for tag in tags: + regex = self.kw_retry_regex if token == "KEYWORD" else self.test_retry_regex + retry_kw = re.match(regex, tag) + if not retry_kw: + continue + return int(retry_kw.group(1)) + return 0 + + def set_loglevel(self) -> None: + """ + Custom function to set robot log level correctly. + """ + if BuiltIn()._context.output.log_level.level == self.log_level: + return + self.initial_log_level = BuiltIn()._context.output.set_log_level(self.log_level) + BuiltIn()._namespace.variables.set_global("${LOG_LEVEL}", self.log_level) + if BuiltIn()._context.output.log_level.level != self.log_level: + logger.warn("Setting log level failed!") + + def reset_loglevel(self) -> None: + """ + Custom function to reset robot log level correctly. + """ + _initial_log_level = BuiltIn()._context.output.initial_log_level + if BuiltIn()._context.output.log_level.level == _initial_log_level: + return + BuiltIn()._context.output.set_log_level(_initial_log_level) + BuiltIn()._namespace.variables.set_global("${LOG_LEVEL}", _initial_log_level) + if BuiltIn()._context.output.log_level.level != self.initial_log_level: + logger.warn("Resetting log level failed!") + -class RetryMerger(ResultVisitor): - def __init__(self, retried_tests, keep_retried_tests=False): +class RetryMerger(ResultVisitor): # type: ignore[misc] + def __init__( + self, + retried_tests: list[str], + keep_retried_tests: bool = False, + warn_on_test_retry: bool = True, + ): self.retried_tests = retried_tests self.keep_retried_tests = keep_retried_tests - self.test_ids = {} + self.warn_on_test_retry = warn_on_test_retry + self.test_ids: dict[str, str] = {} - def start_suite(self, suite): + def start_suite(self, suite: ModelTestSuite) -> None: if self.keep_retried_tests: return test_dict = {} @@ -117,16 +262,16 @@ def start_suite(self, suite): test_dict[test.name] = test suite.tests = list(test_dict.values()) - def end_suite(self, suite): + def end_suite(self, suite: ModelTestSuite) -> None: for test in suite.tests: if test.longname in self.retried_tests: self.test_ids[test.name] = test.id - def start_errors(self, errors): + def start_errors(self, errors: ModelError) -> None: messages = [] retry_messages = {} for message in errors.messages: - if message.level == "WARN": + if message.level == "WARN" and self.warn_on_test_retry: pattern = re.compile( r"Retry (?P\d+)/(?P\d+) of test '(?P.+)':" ) @@ -145,9 +290,9 @@ def start_errors(self, errors): messages + list(retry_messages.values()), key=lambda m: m.timestamp ) - def _get_test_link(self, test_name): + def _get_test_link(self, test_name: str) -> str: test_id = self.test_ids.get(test_name) - link = ( + return ( f"