From d40e4219fe3f3d6634873c70c22187b31e12443c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B8=D0=B2=D0=BE=D0=B5=D0=B2=20=D0=9D=D0=B8=D0=BA?= =?UTF-8?q?=D0=B8=D1=82=D0=B0?= Date: Mon, 25 May 2026 20:31:50 +0300 Subject: [PATCH 01/11] feat: modify fault parameters manually --- src/imgtests/exec/loaders/kirk.py | 4 +- src/imgtests/suites/fault_injection.py | 438 +++++++++++-------------- 2 files changed, 192 insertions(+), 250 deletions(-) diff --git a/src/imgtests/exec/loaders/kirk.py b/src/imgtests/exec/loaders/kirk.py index 7d32d521..e58860d3 100644 --- a/src/imgtests/exec/loaders/kirk.py +++ b/src/imgtests/exec/loaders/kirk.py @@ -97,7 +97,7 @@ def _validate_fault_probability(fault_prob: int) -> None: err_msg = f"fault_probability must be in range 0..{MAX_FAULT_PROBABILITY}." raise ValueError(err_msg) - def _ensure_debugfs(self) -> ExecResult: + def ensure_debugfs(self) -> ExecResult: """Ensures that debugfs is created and mounted.""" debugfs_path = str(DEBUGFS_MOUNTPOINT) result = common_run_command(("sudo", "mkdir", "-p", debugfs_path), self.ssh_client) @@ -154,7 +154,7 @@ def run( # noqa: PLR0911, PLR0913 if fault_prob is not None: self._validate_fault_probability(fault_prob) - debugfs_res = self._ensure_debugfs() + debugfs_res = self.ensure_debugfs() if debugfs_res.returncode: return debugfs_res, None diff --git a/src/imgtests/suites/fault_injection.py b/src/imgtests/suites/fault_injection.py index e6a4f3d9..abe01d7c 100644 --- a/src/imgtests/suites/fault_injection.py +++ b/src/imgtests/suites/fault_injection.py @@ -1,9 +1,11 @@ +import logging import random from datetime import UTC, datetime from pathlib import Path from time import sleep from typing import TYPE_CHECKING +from imgtests.exec.exec import ExecResult, SSHClient, common_run_command from imgtests.exec.loaders import Chaosblade, ChaosResponse, Kirk, Perf, StressNg from imgtests.exec.osinfo import get_os_release from imgtests.exec.user_commands import MkDir @@ -17,6 +19,9 @@ from imgtests.exec.exec import ExecResult, SSHClient +DEBUG_FS_PATH = "/sys/kernel/debug/" +logger = logging.getLogger(__name__) + class FaultInjectionEnduranceTest(AbstractRunnableTimeLimitedTest): def __init__(self, timeout: int, iterations: int = 4) -> None: @@ -85,10 +90,6 @@ def __init__(self, timeout: int) -> None: timeout, ) - @property - def kirk_suites(self) -> tuple[str, str]: - return ("dio", "sched") - @property def chaosblade_suites(self) -> tuple[str, str]: return ("create_disk_exp", "create_cpu_exp") @@ -111,76 +112,58 @@ def _run( mkdir([tmp_dir]) timeout_suite = calc_subtest_timeout( timeout, - len(self.kirk_suites) * len(self.fault_probs), + len(self.chaosblade_suites) * len(self.fault_probs), 60, ) chaosblade = Chaosblade(client) - kirk = Kirk(client) - if not is_kirk_suites_available(kirk, self.kirk_suites): - self.logger.warning("Kirk suite not available for the image with LTP.") - return TestResult(status=TestStatus.SKIPPED) - for fault_prob in self.fault_probs: - self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) - for kirk_suite, chaosblade_suite in zip( - self.kirk_suites, - self.chaosblade_suites, - strict=True, - ): - started_at = datetime.now(UTC) - kirk_future = executor.submit( - kirk.run, - scenarios=[kirk_suite], - timeout=timeout_suite, - fault_prob=fault_prob, - fault_interval=5, - ) - sleep(1) - - chaosblade_future = self._create_chaosblade_future( - executor, - chaosblade, - chaosblade_suite, - tmp_dir, - timeout_suite, - ) - result, chaosblade_result = chaosblade_future.result() - status = TestStatus.PASSED - - if chaosblade_result.success and isinstance(chaosblade_result.result, str): - future = executor.submit( - chaosblade.await_exp_result, - chaosblade_result.result, - ) - while not future.done(): - _, chaosblade_status = chaosblade.get_exp_status(chaosblade_result.result) - if chaosblade_status.result["Status"] == "Error": - status = TestStatus.FAILED - break - sleep(1) - future.result() - else: - status = TestStatus.FAILED - yield TestResult( - metrics=chaosblade_result, - command=" ".join(result.cmd), - started_at=started_at, - status=status, - ) - - result, metrics_path = kirk_future.result() - if metrics_path: - yield TestResult( - command=" ".join(result.cmd), - metrics=kirk.metrics_to_json(metrics_path), - started_at=started_at, - status=TestStatus.PASSED, + try: + for fault_prob in self.fault_probs: + self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) + for chaosblade_suite in self.chaosblade_suites: + started_at = datetime.now(UTC) + result = change_fault_parameters(client, fault_prob, 5) + if result.returncode: + yield TestResult( + command=" ".join(result.cmd), + started_at=started_at, + status=TestStatus.FAILED, + ) + return + + chaosblade_future = self._create_chaosblade_future( + executor, + chaosblade, + chaosblade_suite, + tmp_dir, + timeout_suite, ) - else: + result, chaosblade_result = chaosblade_future.result() + status = TestStatus.PASSED + + if chaosblade_result.success and isinstance(chaosblade_result.result, str): + future = executor.submit( + chaosblade.await_exp_result, + chaosblade_result.result, + ) + while not future.done(): + _, chaosblade_status = chaosblade.get_exp_status( + chaosblade_result.result, + ) + if chaosblade_status.result["Status"] == "Error": + status = TestStatus.FAILED + break + sleep(1) + future.result() + else: + status = TestStatus.FAILED yield TestResult( + metrics=chaosblade_result, command=" ".join(result.cmd), started_at=started_at, - status=TestStatus.FAILED, + status=status, ) + finally: + change_fault_parameters(client, 0, 1) def _create_chaosblade_future( self, @@ -218,10 +201,6 @@ def __init__(self, timeout: int) -> None: timeout, ) - @property - def kirk_suites(self) -> tuple[str, str]: - return ("dio", "syscalls") - @property def stress_ng_suites(self) -> tuple[dict[str, str | int], dict[str, str | int]]: return ( @@ -243,72 +222,46 @@ def _run( self.logger.warning("Skipping test due to fault injection is supported on poky.") return TestResult(status=TestStatus.SKIPPED) - kirk = Kirk(client) - if not is_kirk_suites_available(kirk, self.kirk_suites): - self.logger.warning("Kirk suite not available for the image with LTP.") - return TestResult(status=TestStatus.SKIPPED) - stress_ng = StressNg(client) - timeout_suite = calc_subtest_timeout( - timeout, - len(self.kirk_suites) * len(self.fault_probs), - 60, - ) - - for fault_prob in self.fault_probs: - self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) - for kirk_suite, stress_ng_suite in zip( - self.kirk_suites, - self.stress_ng_suites, - strict=True, - ): - started_at = datetime.now(UTC) - kirk_future = executor.submit( - kirk.run, - scenarios=[kirk_suite], - timeout=timeout_suite, - fault_prob=fault_prob, - fault_interval=5, - ) - sleep(1) - - stress_ng_future = executor.submit( - stress_ng.run, - timeout=timeout_suite, - **stress_ng_suite, - ) - result, metrics = stress_ng_future.result() - - if result.returncode == stress_ng.INCORRECT_OPT_OR_FATAL_ISSUE_CODE: - self.logger.error("stress-ng test BROKEN") - yield TestResult(status=TestStatus.BROKEN) - return - elif result.returncode: - self.logger.error("stress-ng test FAILED") - yield TestResult(status=TestStatus.FAILED) - return + timeout_suite = (timeout // (len(self.stress_ng_suites) * len(self.fault_probs))) + 1 + try: + for fault_prob in self.fault_probs: + self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) + for stress_ng_suite in self.stress_ng_suites: + started_at = datetime.now(UTC) + result = change_fault_parameters(client, fault_prob, 5) + if result.returncode: + yield TestResult( + command=" ".join(result.cmd), + started_at=started_at, + status=TestStatus.FAILED, + ) + return + + stress_ng_future = executor.submit( + stress_ng.run, + timeout=timeout_suite, + **stress_ng_suite, + ) + result, metrics = stress_ng_future.result() - yield TestResult( - metrics=stress_ng.metrics_to_json(metrics), - command=" ".join(result.cmd), - started_at=started_at, - status=TestStatus.PASSED, - ) + if result.returncode == stress_ng.INCORRECT_OPT_OR_FATAL_ISSUE_CODE: + self.logger.error("stress-ng test BROKEN") + yield TestResult(status=TestStatus.BROKEN) + return + elif result.returncode: + self.logger.error("stress-ng test FAILED") + yield TestResult(status=TestStatus.FAILED) + return - result, metrics_path = kirk_future.result() - if metrics_path: yield TestResult( + metrics=stress_ng.metrics_to_json(metrics), command=" ".join(result.cmd), - metrics=kirk.metrics_to_json(metrics_path), started_at=started_at, status=TestStatus.PASSED, ) - else: - yield TestResult( - command=" ".join(result.cmd), - started_at=started_at, - status=TestStatus.FAILED, - ) + finally: + change_fault_parameters(client, 0, 1) class FaultInjectionPerfTest(AbstractRunnableTimeLimitedTest): @@ -319,10 +272,6 @@ def __init__(self, timeout: int) -> None: timeout, ) - @property - def kirk_suites(self) -> tuple[str, str]: - return ("dio", "syscalls") - @property def perf_suites(self) -> tuple[str, str]: return ("sched", "syscall") @@ -341,69 +290,43 @@ def _run( self.logger.warning("Skipping test due to fault injection is supported on poky.") return TestResult(status=TestStatus.SKIPPED) - kirk = Kirk(client) - if not is_kirk_suites_available(kirk, self.kirk_suites): - self.logger.warning("Kirk suite not available for the image with LTP.") - return TestResult(status=TestStatus.SKIPPED) - perf = Perf(client) - timeout_suite = calc_subtest_timeout( - timeout, - len(self.kirk_suites) * len(self.fault_probs), - 60, - ) - - for fault_prob in self.fault_probs: - self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) - for kirk_suite, perf_suite in zip( - self.kirk_suites, - self.perf_suites, - strict=True, - ): - started_at = datetime.now(UTC) - kirk_future = executor.submit( - kirk.run, - scenarios=[kirk_suite], - timeout=timeout_suite, - fault_prob=fault_prob, - fault_interval=5, - ) - sleep(1) - - perf_future = executor.submit( - perf.bench, - collection=perf_suite, - ) - - result, perf_metrics = perf_future.result() - if result.returncode: - yield TestResult( - command=" ".join(result.cmd), - started_at=started_at, - status=TestStatus.FAILED, - ) - elif perf_metrics: - yield TestResult( - command=" ".join(result.cmd), - metrics=perf.metrics_to_json(perf_metrics), - started_at=started_at, - status=TestStatus.PASSED, + timeout_suite = (timeout // (len(self.perf_suites) * len(self.fault_probs))) + 1 + try: + for fault_prob in self.fault_probs: + self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) + for perf_suite in self.perf_suites: + started_at = datetime.now(UTC) + result = change_fault_parameters(client, fault_prob, 5) + if result.returncode: + yield TestResult( + command=" ".join(result.cmd), + started_at=started_at, + status=TestStatus.FAILED, + ) + return + + perf_future = executor.submit( + perf.bench, + collection=perf_suite, ) - result, metrics_path = kirk_future.result() - if metrics_path: - yield TestResult( - command=" ".join(result.cmd), - metrics=kirk.metrics_to_json(metrics_path), - started_at=started_at, - status=TestStatus.PASSED, - ) - else: - yield TestResult( - command=" ".join(result.cmd), - started_at=started_at, - status=TestStatus.FAILED, - ) + result, perf_metrics = perf_future.result() + if result.returncode: + yield TestResult( + command=" ".join(result.cmd), + started_at=started_at, + status=TestStatus.FAILED, + ) + elif perf_metrics: + yield TestResult( + command=" ".join(result.cmd), + metrics=perf.metrics_to_json(perf_metrics), + started_at=started_at, + status=TestStatus.PASSED, + ) + finally: + change_fault_parameters(client, 0, 1) class FaultInjectionFioTest(AbstractRunnableTimeLimitedTest): @@ -414,10 +337,6 @@ def __init__(self, timeout: int) -> None: timeout, ) - @property - def kirk_suites(self) -> tuple[str, str]: - return ("dio", "syscalls") - @property def fio_suites(self) -> tuple[FioWorkload, ...]: return ( @@ -431,7 +350,7 @@ def fault_probs(self) -> tuple[int, ...]: def _run( self, - executor: ThreadPoolExecutor, + executor: ThreadPoolExecutor, # noqa: ARG002 client: SSHClient | None, timeout: int, ) -> Iterable[TestResult]: @@ -439,65 +358,88 @@ def _run( self.logger.warning("Skipping test due to fault injection is supported on poky.") return TestResult(status=TestStatus.SKIPPED) - kirk = Kirk(client) - if not is_kirk_suites_available(kirk, self.kirk_suites): - self.logger.warning("Kirk suite not available for the image with LTP.") - return TestResult(status=TestStatus.SKIPPED) - - timeout_suite = calc_subtest_timeout( - timeout, - len(self.kirk_suites) * len(self.fault_probs), - 60, - ) - - for fault_prob in self.fault_probs: - self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) - for kirk_suite, fio_suite in zip( - self.kirk_suites, - self.fio_suites, - strict=True, - ): - started_at = datetime.now(UTC) - kirk_future = executor.submit( - kirk.run, - scenarios=[kirk_suite], - timeout=timeout_suite, - fault_prob=fault_prob, - fault_interval=5, - ) - sleep(1) - - fio_config = FioSuiteConfig( - suite="fault_injection", - duration_sec=timeout_suite, - results_dir=Path().home() / "fio", - workloads=(fio_suite,), - ) - - fio = FioSuite(client, fio_config).run() - yield from fio - - result, metrics_path = kirk_future.result() - if metrics_path: - yield TestResult( - command=" ".join(result.cmd), - metrics=kirk.metrics_to_json(metrics_path), - started_at=started_at, - status=TestStatus.PASSED, - ) - else: - yield TestResult( - command=" ".join(result.cmd), - started_at=started_at, - status=TestStatus.FAILED, + timeout_suite = (timeout // (len(self.fio_suites) * len(self.fault_probs))) + 1 + try: + for fault_prob in self.fault_probs: + self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) + for fio_suite in self.fio_suites: + started_at = datetime.now(UTC) + result = change_fault_parameters(client, fault_prob, 5) + if result.returncode: + yield TestResult( + command=" ".join(result.cmd), + started_at=started_at, + status=TestStatus.FAILED, + ) + return + + fio_config = FioSuiteConfig( + suite="fault_injection", + duration_sec=timeout_suite, + results_dir=Path().home() / "fio", + workloads=(fio_suite,), ) + fio = FioSuite(client, fio_config).run() + yield from fio + finally: + change_fault_parameters(client, 0, 1) + self.logger.info("Reset fault injection parameters to default.") + def is_fault_injection_available(client: SSHClient | None) -> bool: os_id = get_os_release(client).id return not (os_id and os_id != Distro.POKY.value) +def change_fault_parameters( + client: SSHClient, + fault_probability: int, + fault_interval: int, +) -> ExecResult: + result = Kirk(client).ensure_debugfs() + if result.returncode: + return result + + result = common_run_command(["ls", DEBUG_FS_PATH], ssh_client=client) + if result.returncode: + logger.error("Failed to list debugfs directory.") + return result + + dirs = [ + i + for i in result.stdout.splitlines() + if ("fail" in i or "fault" in i) and i != "fault_around_bytes" + ] + if not dirs: + logger.warning("No fault-injection debugfs entries found under %s", DEBUG_FS_PATH) + return result + + times = -1 if fault_probability > 0 else 1 + last_result = result + for directory in dirs: + dir_path = DEBUG_FS_PATH + directory + updates = ( + ("interval", fault_interval), + ("space", 0), + ("times", times), + ("probability", fault_probability), + ) + + for parameter, value in updates: + write_cmd = ["echo", f"{value}", ">", f"{dir_path}/{parameter}"] + result = common_run_command(write_cmd, ssh_client=client) + last_result = result + if result.returncode: + logger.error( + "Failed to update fault-injection parameter %s in %s", + parameter, + dir_path, + ) + return result + return last_result + + def is_kirk_suites_available(kirk: Kirk, required_suites: tuple[str, ...]) -> bool: available_suites = kirk.list_suites() return all(suite in available_suites for suite in required_suites) From ce7e6ec609eb19f39a90dc738680287833a5050d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9D=D0=B8=D0=BA=D0=B8=D1=82=D0=B0=20=D0=9F=D0=B8=D0=B2?= =?UTF-8?q?=D0=BE=D0=B5=D0=B2?= Date: Mon, 25 May 2026 20:59:51 +0300 Subject: [PATCH 02/11] refactor: remove leftover logger.info --- src/imgtests/suites/fault_injection.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/imgtests/suites/fault_injection.py b/src/imgtests/suites/fault_injection.py index a7a23731..1350d2b0 100644 --- a/src/imgtests/suites/fault_injection.py +++ b/src/imgtests/suites/fault_injection.py @@ -383,7 +383,6 @@ def _run( yield from fio finally: change_fault_parameters(client, 0, 1) - self.logger.info("Reset fault injection parameters to default.") def is_fault_injection_available(client: SSHClient | None) -> bool: From 45c5a855c5532c560fd5652ae637ba81e252e2fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B8=D0=B2=D0=BE=D0=B5=D0=B2=20=D0=9D=D0=B8=D0=BA?= =?UTF-8?q?=D0=B8=D1=82=D0=B0?= Date: Fri, 12 Jun 2026 12:57:52 +0300 Subject: [PATCH 03/11] refactor: move kirk function to class --- src/imgtests/exec/loaders/kirk.py | 4 ++++ src/imgtests/suites/fault_injection.py | 7 +------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/imgtests/exec/loaders/kirk.py b/src/imgtests/exec/loaders/kirk.py index e58860d3..a97007f4 100644 --- a/src/imgtests/exec/loaders/kirk.py +++ b/src/imgtests/exec/loaders/kirk.py @@ -90,6 +90,10 @@ def list_suites( return tuple(line.strip() for line in res.stdout.splitlines() if line.strip()) + def is_suites_available(self, required_suites: tuple[str, ...]) -> bool: + available_suites = self.list_suites() + return all(suite in available_suites for suite in required_suites) + @staticmethod def _validate_fault_probability(fault_prob: int) -> None: """Checks if fault injection probability is in between borders.""" diff --git a/src/imgtests/suites/fault_injection.py b/src/imgtests/suites/fault_injection.py index 1350d2b0..0e0425ad 100644 --- a/src/imgtests/suites/fault_injection.py +++ b/src/imgtests/suites/fault_injection.py @@ -46,7 +46,7 @@ def _run( return TestResult(status=TestStatus.SKIPPED) kirk = Kirk(client) - if not is_kirk_suites_available(kirk, self.kirk_suites): + if not kirk.is_suites_available(self.kirk_suites): self.logger.warning("Kirk suite not available for the image with LTP.") return TestResult(status=TestStatus.SKIPPED) @@ -436,8 +436,3 @@ def change_fault_parameters( ) return result return last_result - - -def is_kirk_suites_available(kirk: Kirk, required_suites: tuple[str, ...]) -> bool: - available_suites = kirk.list_suites() - return all(suite in available_suites for suite in required_suites) From 37a1447a3b91938e26563df62afe33a0a0875c35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B8=D0=B2=D0=BE=D0=B5=D0=B2=20=D0=9D=D0=B8=D0=BA?= =?UTF-8?q?=D0=B8=D1=82=D0=B0?= Date: Fri, 12 Jun 2026 13:17:32 +0300 Subject: [PATCH 04/11] refactor: move debugfs functions to separate file --- src/imgtests/exec/debugfs.py | 95 ++++++++++++++++++++++++++ src/imgtests/exec/loaders/kirk.py | 38 +---------- src/imgtests/suites/fault_injection.py | 53 +------------- 3 files changed, 101 insertions(+), 85 deletions(-) create mode 100644 src/imgtests/exec/debugfs.py diff --git a/src/imgtests/exec/debugfs.py b/src/imgtests/exec/debugfs.py new file mode 100644 index 00000000..67d635d6 --- /dev/null +++ b/src/imgtests/exec/debugfs.py @@ -0,0 +1,95 @@ +import logging +from pathlib import Path +from typing import TYPE_CHECKING + +from imgtests.exec.exec import ExecResult, SSHClient, common_run_command +from imgtests.exec.loaders import Kirk + +if TYPE_CHECKING: + from imgtests.exec.exec import ExecResult, SSHClient + +DEBUGFS_MOUNTPOINT = Path("/sys/kernel/debug") +DEBUG_FS_PATH = "/sys/kernel/debug/" +MAX_FAULT_PROBABILITY = 100 + +logger = logging.getLogger(__name__) + + +def ensure_debugfs(self) -> ExecResult: + """Ensures that debugfs is created and mounted.""" + debugfs_path = str(DEBUGFS_MOUNTPOINT) + result = common_run_command(("sudo", "mkdir", "-p", debugfs_path), self.ssh_client) + if result.returncode: + return result + mount_pattern = f"[[:space:]]{debugfs_path}[[:space:]]debugfs[[:space:]]" + result = common_run_command( + ("sudo", "grep", "-qs", mount_pattern, "/proc/mounts"), + self.ssh_client, + ) + if result.returncode == 0 or result.returncode != 1: + return result + logger.info("Mounting debugfs to '%s'.", debugfs_path) + result = common_run_command( + ("sudo", "mount", "-t", "debugfs", "debugfs", debugfs_path), + self.ssh_client, + ) + + if result.returncode: + logger.info("Unmounting debugfs from '%s'.", debugfs_path) + common_run_command(("sudo", "umount", debugfs_path), self.ssh_client) + return result + + +def validate_fault_probability(fault_prob: int) -> None: + """Checks if fault injection probability is in between borders.""" + if not 0 <= fault_prob <= MAX_FAULT_PROBABILITY: + err_msg = f"fault_probability must be in range 0..{MAX_FAULT_PROBABILITY}." + raise ValueError(err_msg) + + +def change_fault_parameters( + client: SSHClient, + fault_probability: int, + fault_interval: int, +) -> ExecResult: + result = Kirk(client).ensure_debugfs() + if result.returncode: + return result + + result = common_run_command(["ls", DEBUG_FS_PATH], ssh_client=client) + if result.returncode: + logger.error("Failed to list debugfs directory.") + return result + + dirs = [ + i + for i in result.stdout.splitlines() + if ("fail" in i or "fault" in i) and i != "fault_around_bytes" + ] + if not dirs: + logger.warning("No fault-injection debugfs entries found under %s", DEBUG_FS_PATH) + return result + + times = -1 if fault_probability > 0 else 1 + last_result = result + for directory in dirs: + dir_path = DEBUG_FS_PATH + directory + updates = ( + ("interval", fault_interval), + ("space", 0), + ("times", times), + ("probability", fault_probability), + ) + + for parameter, value in updates: + write_cmd = ["echo", f"{value}", ">", f"{dir_path}/{parameter}"] + result = common_run_command(write_cmd, ssh_client=client) + last_result = result + if result.returncode: + logger.error( + "Failed to update fault-injection parameter %s in %s", + parameter, + dir_path, + ) + return result + return last_result diff --git a/src/imgtests/exec/loaders/kirk.py b/src/imgtests/exec/loaders/kirk.py index a97007f4..b7e38273 100644 --- a/src/imgtests/exec/loaders/kirk.py +++ b/src/imgtests/exec/loaders/kirk.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any from imgtests.exec.base_util import GenericUtil +from imgtests.exec.debugfs import ensure_debugfs, validate_fault_probability from imgtests.exec.exec import ExecResult, SSHClient, common_run_command from imgtests.exec.osinfo import get_os_release from imgtests.exec.pkgmgrs.zypper import Zypper @@ -19,8 +20,6 @@ logger = logging.getLogger(__name__) DEFAULT_LTP_RESULTS_DIR = Path("/var/tmp/ltp-results") # noqa: S108 -DEBUGFS_MOUNTPOINT = Path("/sys/kernel/debug") -MAX_FAULT_PROBABILITY = 100 class Kirk(GenericUtil): @@ -94,37 +93,6 @@ def is_suites_available(self, required_suites: tuple[str, ...]) -> bool: available_suites = self.list_suites() return all(suite in available_suites for suite in required_suites) - @staticmethod - def _validate_fault_probability(fault_prob: int) -> None: - """Checks if fault injection probability is in between borders.""" - if not 0 <= fault_prob <= MAX_FAULT_PROBABILITY: - err_msg = f"fault_probability must be in range 0..{MAX_FAULT_PROBABILITY}." - raise ValueError(err_msg) - - def ensure_debugfs(self) -> ExecResult: - """Ensures that debugfs is created and mounted.""" - debugfs_path = str(DEBUGFS_MOUNTPOINT) - result = common_run_command(("sudo", "mkdir", "-p", debugfs_path), self.ssh_client) - if result.returncode: - return result - mount_pattern = f"[[:space:]]{debugfs_path}[[:space:]]debugfs[[:space:]]" - result = common_run_command( - ("sudo", "grep", "-qs", mount_pattern, "/proc/mounts"), - self.ssh_client, - ) - if result.returncode == 0 or result.returncode != 1: - return result - logger.info("Mounting debugfs to '%s'.", debugfs_path) - result = common_run_command( - ("sudo", "mount", "-t", "debugfs", "debugfs", debugfs_path), - self.ssh_client, - ) - - if result.returncode: - logger.info("Unmounting debugfs from '%s'.", debugfs_path) - common_run_command(("sudo", "umount", debugfs_path), self.ssh_client) - return result - def run( # noqa: PLR0911, PLR0913 self, scenarios: Iterable[str], @@ -156,9 +124,9 @@ def run( # noqa: PLR0911, PLR0913 raise ValueError(scenarios_empty_msg) if fault_prob is not None: - self._validate_fault_probability(fault_prob) + validate_fault_probability(fault_prob) - debugfs_res = self.ensure_debugfs() + debugfs_res = ensure_debugfs() if debugfs_res.returncode: return debugfs_res, None diff --git a/src/imgtests/suites/fault_injection.py b/src/imgtests/suites/fault_injection.py index 0e0425ad..f8105d5f 100644 --- a/src/imgtests/suites/fault_injection.py +++ b/src/imgtests/suites/fault_injection.py @@ -4,7 +4,8 @@ from time import sleep from typing import TYPE_CHECKING -from imgtests.exec.exec import ExecResult, SSHClient, common_run_command +from imgtests.exec.debugfs import change_fault_parameters +from imgtests.exec.exec import ExecResult, SSHClient from imgtests.exec.loaders import Chaosblade, ChaosResponse, Kirk, Perf, StressNg from imgtests.exec.osinfo import get_os_release from imgtests.exec.user_commands import MkDir @@ -18,7 +19,7 @@ from imgtests.exec.exec import ExecResult, SSHClient -DEBUG_FS_PATH = "/sys/kernel/debug/" + logger = logging.getLogger(__name__) @@ -388,51 +389,3 @@ def _run( def is_fault_injection_available(client: SSHClient | None) -> bool: os_id = get_os_release(client).id return not (os_id and os_id != Distro.POKY.value) - - -def change_fault_parameters( - client: SSHClient, - fault_probability: int, - fault_interval: int, -) -> ExecResult: - result = Kirk(client).ensure_debugfs() - if result.returncode: - return result - - result = common_run_command(["ls", DEBUG_FS_PATH], ssh_client=client) - if result.returncode: - logger.error("Failed to list debugfs directory.") - return result - - dirs = [ - i - for i in result.stdout.splitlines() - if ("fail" in i or "fault" in i) and i != "fault_around_bytes" - ] - if not dirs: - logger.warning("No fault-injection debugfs entries found under %s", DEBUG_FS_PATH) - return result - - times = -1 if fault_probability > 0 else 1 - last_result = result - for directory in dirs: - dir_path = DEBUG_FS_PATH + directory - updates = ( - ("interval", fault_interval), - ("space", 0), - ("times", times), - ("probability", fault_probability), - ) - - for parameter, value in updates: - write_cmd = ["echo", f"{value}", ">", f"{dir_path}/{parameter}"] - result = common_run_command(write_cmd, ssh_client=client) - last_result = result - if result.returncode: - logger.error( - "Failed to update fault-injection parameter %s in %s", - parameter, - dir_path, - ) - return result - return last_result From 642c0c4600d8bf802d1367fec7d7c37c8d61a164 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B8=D0=B2=D0=BE=D0=B5=D0=B2=20=D0=9D=D0=B8=D0=BA?= =?UTF-8?q?=D0=B8=D1=82=D0=B0?= Date: Fri, 12 Jun 2026 14:21:14 +0300 Subject: [PATCH 05/11] fix: remove self from the function --- src/imgtests/exec/debugfs.py | 13 ++++++------- src/imgtests/exec/loaders/kirk.py | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/imgtests/exec/debugfs.py b/src/imgtests/exec/debugfs.py index 67d635d6..00af3019 100644 --- a/src/imgtests/exec/debugfs.py +++ b/src/imgtests/exec/debugfs.py @@ -3,7 +3,6 @@ from typing import TYPE_CHECKING from imgtests.exec.exec import ExecResult, SSHClient, common_run_command -from imgtests.exec.loaders import Kirk if TYPE_CHECKING: from imgtests.exec.exec import ExecResult, SSHClient @@ -15,28 +14,28 @@ logger = logging.getLogger(__name__) -def ensure_debugfs(self) -> ExecResult: +def ensure_debugfs(ssh_client: SSHClient) -> ExecResult: """Ensures that debugfs is created and mounted.""" debugfs_path = str(DEBUGFS_MOUNTPOINT) - result = common_run_command(("sudo", "mkdir", "-p", debugfs_path), self.ssh_client) + result = common_run_command(("sudo", "mkdir", "-p", debugfs_path), ssh_client) if result.returncode: return result mount_pattern = f"[[:space:]]{debugfs_path}[[:space:]]debugfs[[:space:]]" result = common_run_command( ("sudo", "grep", "-qs", mount_pattern, "/proc/mounts"), - self.ssh_client, + ssh_client, ) if result.returncode == 0 or result.returncode != 1: return result logger.info("Mounting debugfs to '%s'.", debugfs_path) result = common_run_command( ("sudo", "mount", "-t", "debugfs", "debugfs", debugfs_path), - self.ssh_client, + ssh_client, ) if result.returncode: logger.info("Unmounting debugfs from '%s'.", debugfs_path) - common_run_command(("sudo", "umount", debugfs_path), self.ssh_client) + common_run_command(("sudo", "umount", debugfs_path), ssh_client) return result @@ -52,7 +51,7 @@ def change_fault_parameters( fault_probability: int, fault_interval: int, ) -> ExecResult: - result = Kirk(client).ensure_debugfs() + result = ensure_debugfs(client) if result.returncode: return result diff --git a/src/imgtests/exec/loaders/kirk.py b/src/imgtests/exec/loaders/kirk.py index b7e38273..8bfaed7c 100644 --- a/src/imgtests/exec/loaders/kirk.py +++ b/src/imgtests/exec/loaders/kirk.py @@ -126,7 +126,7 @@ def run( # noqa: PLR0911, PLR0913 if fault_prob is not None: validate_fault_probability(fault_prob) - debugfs_res = ensure_debugfs() + debugfs_res = ensure_debugfs(self.ssh_client) if debugfs_res.returncode: return debugfs_res, None From 5dba9de994fd7bb8bb8a6e745e65f0048643ed47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B8=D0=B2=D0=BE=D0=B5=D0=B2=20=D0=9D=D0=B8=D0=BA?= =?UTF-8?q?=D0=B8=D1=82=D0=B0?= Date: Sun, 14 Jun 2026 16:04:58 +0300 Subject: [PATCH 06/11] chore: add comments and fix debugfs path --- src/imgtests/exec/debugfs.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/imgtests/exec/debugfs.py b/src/imgtests/exec/debugfs.py index 00af3019..f237dae0 100644 --- a/src/imgtests/exec/debugfs.py +++ b/src/imgtests/exec/debugfs.py @@ -1,5 +1,4 @@ import logging -from pathlib import Path from typing import TYPE_CHECKING from imgtests.exec.exec import ExecResult, SSHClient, common_run_command @@ -7,8 +6,7 @@ if TYPE_CHECKING: from imgtests.exec.exec import ExecResult, SSHClient -DEBUGFS_MOUNTPOINT = Path("/sys/kernel/debug") -DEBUG_FS_PATH = "/sys/kernel/debug/" +DEBUGFS_PATH = "/sys/kernel/debug" MAX_FAULT_PROBABILITY = 100 logger = logging.getLogger(__name__) @@ -16,26 +14,25 @@ def ensure_debugfs(ssh_client: SSHClient) -> ExecResult: """Ensures that debugfs is created and mounted.""" - debugfs_path = str(DEBUGFS_MOUNTPOINT) - result = common_run_command(("sudo", "mkdir", "-p", debugfs_path), ssh_client) + result = common_run_command(("sudo", "mkdir", "-p", DEBUGFS_PATH), ssh_client) if result.returncode: return result - mount_pattern = f"[[:space:]]{debugfs_path}[[:space:]]debugfs[[:space:]]" + mount_pattern = f"[[:space:]]{DEBUGFS_PATH}[[:space:]]debugfs[[:space:]]" result = common_run_command( ("sudo", "grep", "-qs", mount_pattern, "/proc/mounts"), ssh_client, ) if result.returncode == 0 or result.returncode != 1: return result - logger.info("Mounting debugfs to '%s'.", debugfs_path) + logger.info("Mounting debugfs to '%s'.", DEBUGFS_PATH) result = common_run_command( - ("sudo", "mount", "-t", "debugfs", "debugfs", debugfs_path), + ("sudo", "mount", "-t", "debugfs", "debugfs", DEBUGFS_PATH), ssh_client, ) if result.returncode: - logger.info("Unmounting debugfs from '%s'.", debugfs_path) - common_run_command(("sudo", "umount", debugfs_path), ssh_client) + logger.info("Unmounting debugfs from '%s'.", DEBUGFS_PATH) + common_run_command(("sudo", "umount", DEBUGFS_PATH), ssh_client) return result @@ -55,24 +52,26 @@ def change_fault_parameters( if result.returncode: return result - result = common_run_command(["ls", DEBUG_FS_PATH], ssh_client=client) + result = common_run_command(["ls", DEBUGFS_PATH], ssh_client=client) if result.returncode: logger.error("Failed to list debugfs directory.") return result dirs = [ - i - for i in result.stdout.splitlines() - if ("fail" in i or "fault" in i) and i != "fault_around_bytes" + dir_name + for dir_name in result.stdout.splitlines() + if ("fail" in dir_name or "fault" in dir_name) and dir_name != "fault_around_bytes" ] if not dirs: - logger.warning("No fault-injection debugfs entries found under %s", DEBUG_FS_PATH) + logger.warning("No fault-injection debugfs entries found under %s", DEBUGFS_PATH) return result + # When fault injection is enabled, times should be set to -1, so that it would run infinitely + # When fault injection is disabled, times set to default 1 times = -1 if fault_probability > 0 else 1 last_result = result for directory in dirs: - dir_path = DEBUG_FS_PATH + directory + dir_path = DEBUGFS_PATH + "/" + directory updates = ( ("interval", fault_interval), ("space", 0), From 07336a30b6c7c4bcf39c435c95bde170f3971862 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B8=D0=B2=D0=BE=D0=B5=D0=B2=20=D0=9D=D0=B8=D0=BA?= =?UTF-8?q?=D0=B8=D1=82=D0=B0?= Date: Sun, 14 Jun 2026 16:49:44 +0300 Subject: [PATCH 07/11] refactor: add cleanup mixin and error processing in runner --- src/imgtests/exec/pkgmgrs/mixin.py | 12 +- src/imgtests/runner.py | 33 +-- src/imgtests/suites/fault_injection.py | 265 ++++++++++++------------- 3 files changed, 158 insertions(+), 152 deletions(-) diff --git a/src/imgtests/exec/pkgmgrs/mixin.py b/src/imgtests/exec/pkgmgrs/mixin.py index 4397f1c0..46260dd1 100644 --- a/src/imgtests/exec/pkgmgrs/mixin.py +++ b/src/imgtests/exec/pkgmgrs/mixin.py @@ -1,10 +1,13 @@ from typing import TYPE_CHECKING -from imgtests.exec.exec import ExecResult +from imgtests.exec.debugfs import change_fault_parameters +from imgtests.exec.exec import ExecResult, SSHClient from imgtests.exec.osinfo import get_os_release from imgtests.exec.pkgmgrs.zypper import Zypper +from imgtests.planning.base import DefaultCleanupMixin if TYPE_CHECKING: + import logging from collections.abc import Iterable @@ -38,3 +41,10 @@ def _install_packages(self, packages: Iterable[str]) -> ExecResult: stderr=msg, returncode=1, ) + + +class FaultCleanupMixin(DefaultCleanupMixin): + def cleanup(self, client: SSHClient | None, logger: logging.Logger) -> None: + change_fault_parameters(client, 0, 1) + logger.info("Cleaned fault parameters.") + super().cleanup(client, logger) diff --git a/src/imgtests/runner.py b/src/imgtests/runner.py index d9e37f54..ed1401f3 100644 --- a/src/imgtests/runner.py +++ b/src/imgtests/runner.py @@ -328,20 +328,27 @@ def handle_tests( test_instance = test_class else: test_instance = test_class(self.__test_config.test_duration) - for result in test_instance(self._executor, self._client): - self._database.insert_util_run_result( - experiment_id=experiment_id, - # TODO: fill util_type with the correct value - util_type="loader", - # TODO: fill descriptions and adds into TestResult class - description="", - result=result.metrics, - command=result.command, - started_at=result.started_at, - ended_at=result.ended_at, + try: + for result in test_instance(self._executor, self._client): + self._database.insert_util_run_result( + experiment_id=experiment_id, + # TODO: fill util_type with the correct value + util_type="loader", + # TODO: fill descriptions and adds into TestResult class + description="", + result=result.metrics, + command=result.command, + started_at=result.started_at, + ended_at=result.ended_at, + ) + self.__tests_statuses[result.status] += 1 + self.__tests_cnt += 1 + except Exception: + self._logger.exception( + "Test '%s' failed with exception.", + test_instance.description, ) - self.__tests_statuses[result.status] += 1 - self.__tests_cnt += 1 + self.__tests_statuses[TestStatus.BROKEN] += 1 self._collect_system_errors( experiment_id=experiment_id, since=test_started_at, diff --git a/src/imgtests/suites/fault_injection.py b/src/imgtests/suites/fault_injection.py index f8105d5f..de01bfb8 100644 --- a/src/imgtests/suites/fault_injection.py +++ b/src/imgtests/suites/fault_injection.py @@ -8,6 +8,7 @@ from imgtests.exec.exec import ExecResult, SSHClient from imgtests.exec.loaders import Chaosblade, ChaosResponse, Kirk, Perf, StressNg from imgtests.exec.osinfo import get_os_release +from imgtests.exec.pkgmgrs.mixin import FaultCleanupMixin from imgtests.exec.user_commands import MkDir from imgtests.planning import AbstractRunnableTimeLimitedTest, calc_subtest_timeout from imgtests.suites.drive.fio import FIO_RESULTS_DIR, FioSuite, FioSuiteConfig, FioWorkload @@ -82,7 +83,7 @@ def _run( ) -class FaultInjectionChaosbladeTest(AbstractRunnableTimeLimitedTest): +class FaultInjectionChaosbladeTest(FaultCleanupMixin, AbstractRunnableTimeLimitedTest): def __init__(self, timeout: int) -> None: super().__init__( "Chaosblade test with fault injection.", @@ -116,54 +117,51 @@ def _run( 60, ) chaosblade = Chaosblade(client) - try: - for fault_prob in self.fault_probs: - self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) - for chaosblade_suite in self.chaosblade_suites: - started_at = datetime.now(UTC) - result = change_fault_parameters(client, fault_prob, 5) - if result.returncode: - yield TestResult( - command=" ".join(result.cmd), - started_at=started_at, - status=TestStatus.FAILED, - ) - return - - chaosblade_future = self._create_chaosblade_future( - executor, - chaosblade, - chaosblade_suite, - tmp_dir, - timeout_suite, - ) - result, chaosblade_result = chaosblade_future.result() - status = TestStatus.PASSED - - if chaosblade_result.success and isinstance(chaosblade_result.result, str): - future = executor.submit( - chaosblade.await_exp_result, - chaosblade_result.result, - ) - while not future.done(): - _, chaosblade_status = chaosblade.get_exp_status( - chaosblade_result.result, - ) - if chaosblade_status.result["Status"] == "Error": - status = TestStatus.FAILED - break - sleep(1) - future.result() - else: - status = TestStatus.FAILED + for fault_prob in self.fault_probs: + self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) + for chaosblade_suite in self.chaosblade_suites: + started_at = datetime.now(UTC) + result = change_fault_parameters(client, fault_prob, 5) + if result.returncode: yield TestResult( - metrics=chaosblade_result, command=" ".join(result.cmd), started_at=started_at, - status=status, + status=TestStatus.FAILED, + ) + return + + chaosblade_future = self._create_chaosblade_future( + executor, + chaosblade, + chaosblade_suite, + tmp_dir, + timeout_suite, + ) + result, chaosblade_result = chaosblade_future.result() + status = TestStatus.PASSED + + if chaosblade_result.success and isinstance(chaosblade_result.result, str): + future = executor.submit( + chaosblade.await_exp_result, + chaosblade_result.result, ) - finally: - change_fault_parameters(client, 0, 1) + while not future.done(): + _, chaosblade_status = chaosblade.get_exp_status( + chaosblade_result.result, + ) + if chaosblade_status.result["Status"] == "Error": + status = TestStatus.FAILED + break + sleep(1) + future.result() + else: + status = TestStatus.FAILED + yield TestResult( + metrics=chaosblade_result, + command=" ".join(result.cmd), + started_at=started_at, + status=status, + ) def _create_chaosblade_future( self, @@ -193,7 +191,7 @@ def _create_chaosblade_future( raise ValueError(err_msg) -class FaultInjectionStressNgTest(AbstractRunnableTimeLimitedTest): +class FaultInjectionStressNgTest(FaultCleanupMixin, AbstractRunnableTimeLimitedTest): def __init__(self, timeout: int) -> None: super().__init__( "Stress-ng test with fault injection.", @@ -224,47 +222,44 @@ def _run( stress_ng = StressNg(client) timeout_suite = (timeout // (len(self.stress_ng_suites) * len(self.fault_probs))) + 1 - try: - for fault_prob in self.fault_probs: - self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) - for stress_ng_suite in self.stress_ng_suites: - started_at = datetime.now(UTC) - result = change_fault_parameters(client, fault_prob, 5) - if result.returncode: - yield TestResult( - command=" ".join(result.cmd), - started_at=started_at, - status=TestStatus.FAILED, - ) - return - - stress_ng_future = executor.submit( - stress_ng.run, - timeout=timeout_suite, - **stress_ng_suite, - ) - result, metrics = stress_ng_future.result() - - if result.returncode == stress_ng.INCORRECT_OPT_OR_FATAL_ISSUE_CODE: - self.logger.error("stress-ng test BROKEN") - yield TestResult(status=TestStatus.BROKEN) - return - elif result.returncode: - self.logger.error("stress-ng test FAILED") - yield TestResult(status=TestStatus.FAILED) - return - + for fault_prob in self.fault_probs: + self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) + for stress_ng_suite in self.stress_ng_suites: + started_at = datetime.now(UTC) + result = change_fault_parameters(client, fault_prob, 5) + if result.returncode: yield TestResult( - metrics=stress_ng.metrics_to_json(metrics), command=" ".join(result.cmd), started_at=started_at, - status=TestStatus.PASSED, + status=TestStatus.FAILED, ) - finally: - change_fault_parameters(client, 0, 1) + return + + stress_ng_future = executor.submit( + stress_ng.run, + timeout=timeout_suite, + **stress_ng_suite, + ) + result, metrics = stress_ng_future.result() + + if result.returncode == stress_ng.INCORRECT_OPT_OR_FATAL_ISSUE_CODE: + self.logger.error("stress-ng test BROKEN") + yield TestResult(status=TestStatus.BROKEN) + return + elif result.returncode: + self.logger.error("stress-ng test FAILED") + yield TestResult(status=TestStatus.FAILED) + return + + yield TestResult( + metrics=stress_ng.metrics_to_json(metrics), + command=" ".join(result.cmd), + started_at=started_at, + status=TestStatus.PASSED, + ) -class FaultInjectionPerfTest(AbstractRunnableTimeLimitedTest): +class FaultInjectionPerfTest(FaultCleanupMixin, AbstractRunnableTimeLimitedTest): def __init__(self, timeout: int) -> None: super().__init__( "Perf bench test with fault injection.", @@ -292,44 +287,41 @@ def _run( perf = Perf(client) timeout_suite = (timeout // (len(self.perf_suites) * len(self.fault_probs))) + 1 - try: - for fault_prob in self.fault_probs: - self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) - for perf_suite in self.perf_suites: - started_at = datetime.now(UTC) - result = change_fault_parameters(client, fault_prob, 5) - if result.returncode: - yield TestResult( - command=" ".join(result.cmd), - started_at=started_at, - status=TestStatus.FAILED, - ) - return - - perf_future = executor.submit( - perf.bench, - collection=perf_suite, + for fault_prob in self.fault_probs: + self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) + for perf_suite in self.perf_suites: + started_at = datetime.now(UTC) + result = change_fault_parameters(client, fault_prob, 5) + if result.returncode: + yield TestResult( + command=" ".join(result.cmd), + started_at=started_at, + status=TestStatus.FAILED, ) + return - result, perf_metrics = perf_future.result() - if result.returncode: - yield TestResult( - command=" ".join(result.cmd), - started_at=started_at, - status=TestStatus.FAILED, - ) - elif perf_metrics: - yield TestResult( - command=" ".join(result.cmd), - metrics=perf.metrics_to_json(perf_metrics), - started_at=started_at, - status=TestStatus.PASSED, - ) - finally: - change_fault_parameters(client, 0, 1) + perf_future = executor.submit( + perf.bench, + collection=perf_suite, + ) + + result, perf_metrics = perf_future.result() + if result.returncode: + yield TestResult( + command=" ".join(result.cmd), + started_at=started_at, + status=TestStatus.FAILED, + ) + elif perf_metrics: + yield TestResult( + command=" ".join(result.cmd), + metrics=perf.metrics_to_json(perf_metrics), + started_at=started_at, + status=TestStatus.PASSED, + ) -class FaultInjectionFioTest(AbstractRunnableTimeLimitedTest): +class FaultInjectionFioTest(FaultCleanupMixin, AbstractRunnableTimeLimitedTest): def __init__(self, timeout: int) -> None: super().__init__( "Fio test with fault injection.", @@ -359,31 +351,28 @@ def _run( return TestResult(status=TestStatus.SKIPPED) timeout_suite = (timeout // (len(self.fio_suites) * len(self.fault_probs))) + 1 - try: - for fault_prob in self.fault_probs: - self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) - for fio_suite in self.fio_suites: - started_at = datetime.now(UTC) - result = change_fault_parameters(client, fault_prob, 5) - if result.returncode: - yield TestResult( - command=" ".join(result.cmd), - started_at=started_at, - status=TestStatus.FAILED, - ) - return - - fio_config = FioSuiteConfig( - suite="fault_injection", - duration_sec=timeout_suite, - results_dir=FIO_RESULTS_DIR, - workloads=(fio_suite,), + for fault_prob in self.fault_probs: + self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) + for fio_suite in self.fio_suites: + started_at = datetime.now(UTC) + result = change_fault_parameters(client, fault_prob, 5) + if result.returncode: + yield TestResult( + command=" ".join(result.cmd), + started_at=started_at, + status=TestStatus.FAILED, ) + return + + fio_config = FioSuiteConfig( + suite="fault_injection", + duration_sec=timeout_suite, + results_dir=FIO_RESULTS_DIR, + workloads=(fio_suite,), + ) - fio = FioSuite(client, fio_config).run() - yield from fio - finally: - change_fault_parameters(client, 0, 1) + fio = FioSuite(client, fio_config).run() + yield from fio def is_fault_injection_available(client: SSHClient | None) -> bool: From db3ae943c197a211380f3884847d403a293b8a67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B8=D0=B2=D0=BE=D0=B5=D0=B2=20=D0=9D=D0=B8=D0=BA?= =?UTF-8?q?=D0=B8=D1=82=D0=B0?= Date: Sun, 14 Jun 2026 17:19:56 +0300 Subject: [PATCH 08/11] fix: mixin dependencies --- src/imgtests/exec/pkgmgrs/mixin.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/imgtests/exec/pkgmgrs/mixin.py b/src/imgtests/exec/pkgmgrs/mixin.py index 46260dd1..98621539 100644 --- a/src/imgtests/exec/pkgmgrs/mixin.py +++ b/src/imgtests/exec/pkgmgrs/mixin.py @@ -4,7 +4,6 @@ from imgtests.exec.exec import ExecResult, SSHClient from imgtests.exec.osinfo import get_os_release from imgtests.exec.pkgmgrs.zypper import Zypper -from imgtests.planning.base import DefaultCleanupMixin if TYPE_CHECKING: import logging @@ -43,8 +42,10 @@ def _install_packages(self, packages: Iterable[str]) -> ExecResult: ) -class FaultCleanupMixin(DefaultCleanupMixin): +class FaultCleanupMixin: def cleanup(self, client: SSHClient | None, logger: logging.Logger) -> None: change_fault_parameters(client, 0, 1) logger.info("Cleaned fault parameters.") - super().cleanup(client, logger) + default_cleanup = getattr(super(), "cleanup", None) + if default_cleanup: + default_cleanup(client, logger) From a8d367684007e25c052c0891e3693f0fe8cb20dd Mon Sep 17 00:00:00 2001 From: Artanias <43622365+Artanias@users.noreply.github.com> Date: Sun, 14 Jun 2026 22:02:10 +0300 Subject: [PATCH 09/11] refactor: fix type issues and moves FaultCleanupMixin to fault_injection module. --- src/imgtests/exec/debugfs.py | 4 ++-- src/imgtests/exec/pkgmgrs/mixin.py | 13 +------------ src/imgtests/suites/fault_injection.py | 10 +++++++++- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/imgtests/exec/debugfs.py b/src/imgtests/exec/debugfs.py index f237dae0..24bbc08e 100644 --- a/src/imgtests/exec/debugfs.py +++ b/src/imgtests/exec/debugfs.py @@ -12,7 +12,7 @@ logger = logging.getLogger(__name__) -def ensure_debugfs(ssh_client: SSHClient) -> ExecResult: +def ensure_debugfs(ssh_client: SSHClient | None) -> ExecResult: """Ensures that debugfs is created and mounted.""" result = common_run_command(("sudo", "mkdir", "-p", DEBUGFS_PATH), ssh_client) if result.returncode: @@ -44,7 +44,7 @@ def validate_fault_probability(fault_prob: int) -> None: def change_fault_parameters( - client: SSHClient, + client: SSHClient | None, fault_probability: int, fault_interval: int, ) -> ExecResult: diff --git a/src/imgtests/exec/pkgmgrs/mixin.py b/src/imgtests/exec/pkgmgrs/mixin.py index 98621539..4397f1c0 100644 --- a/src/imgtests/exec/pkgmgrs/mixin.py +++ b/src/imgtests/exec/pkgmgrs/mixin.py @@ -1,12 +1,10 @@ from typing import TYPE_CHECKING -from imgtests.exec.debugfs import change_fault_parameters -from imgtests.exec.exec import ExecResult, SSHClient +from imgtests.exec.exec import ExecResult from imgtests.exec.osinfo import get_os_release from imgtests.exec.pkgmgrs.zypper import Zypper if TYPE_CHECKING: - import logging from collections.abc import Iterable @@ -40,12 +38,3 @@ def _install_packages(self, packages: Iterable[str]) -> ExecResult: stderr=msg, returncode=1, ) - - -class FaultCleanupMixin: - def cleanup(self, client: SSHClient | None, logger: logging.Logger) -> None: - change_fault_parameters(client, 0, 1) - logger.info("Cleaned fault parameters.") - default_cleanup = getattr(super(), "cleanup", None) - if default_cleanup: - default_cleanup(client, logger) diff --git a/src/imgtests/suites/fault_injection.py b/src/imgtests/suites/fault_injection.py index de01bfb8..996b5ea9 100644 --- a/src/imgtests/suites/fault_injection.py +++ b/src/imgtests/suites/fault_injection.py @@ -8,7 +8,6 @@ from imgtests.exec.exec import ExecResult, SSHClient from imgtests.exec.loaders import Chaosblade, ChaosResponse, Kirk, Perf, StressNg from imgtests.exec.osinfo import get_os_release -from imgtests.exec.pkgmgrs.mixin import FaultCleanupMixin from imgtests.exec.user_commands import MkDir from imgtests.planning import AbstractRunnableTimeLimitedTest, calc_subtest_timeout from imgtests.suites.drive.fio import FIO_RESULTS_DIR, FioSuite, FioSuiteConfig, FioWorkload @@ -24,6 +23,15 @@ logger = logging.getLogger(__name__) +class FaultCleanupMixin: + def cleanup(self, client: SSHClient | None, logger: logging.Logger) -> None: + change_fault_parameters(client, 0, 1) + logger.info("Cleaned fault parameters.") + default_cleanup = getattr(super(), "cleanup", None) + if default_cleanup: + default_cleanup(client, logger) + + class FaultInjectionEnduranceTest(AbstractRunnableTimeLimitedTest): def __init__(self, timeout: int, iterations: int = 4) -> None: super().__init__( From 3ad57444b8d4528f27f1cc17c3b72a467b7d25d1 Mon Sep 17 00:00:00 2001 From: Artanias <43622365+Artanias@users.noreply.github.com> Date: Sun, 14 Jun 2026 22:06:53 +0300 Subject: [PATCH 10/11] refactor: changes constant usage in the debugfs module. --- src/imgtests/exec/debugfs.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/imgtests/exec/debugfs.py b/src/imgtests/exec/debugfs.py index 24bbc08e..f61f1efa 100644 --- a/src/imgtests/exec/debugfs.py +++ b/src/imgtests/exec/debugfs.py @@ -1,20 +1,21 @@ import logging -from typing import TYPE_CHECKING +from pathlib import Path +from typing import TYPE_CHECKING, Final from imgtests.exec.exec import ExecResult, SSHClient, common_run_command if TYPE_CHECKING: from imgtests.exec.exec import ExecResult, SSHClient -DEBUGFS_PATH = "/sys/kernel/debug" -MAX_FAULT_PROBABILITY = 100 +DEBUGFS_PATH: Final = Path("/sys/kernel/debug") +MAX_FAULT_PROBABILITY: Final = 100 logger = logging.getLogger(__name__) def ensure_debugfs(ssh_client: SSHClient | None) -> ExecResult: """Ensures that debugfs is created and mounted.""" - result = common_run_command(("sudo", "mkdir", "-p", DEBUGFS_PATH), ssh_client) + result = common_run_command(("sudo", "mkdir", "-p", str(DEBUGFS_PATH)), ssh_client) if result.returncode: return result mount_pattern = f"[[:space:]]{DEBUGFS_PATH}[[:space:]]debugfs[[:space:]]" @@ -24,15 +25,15 @@ def ensure_debugfs(ssh_client: SSHClient | None) -> ExecResult: ) if result.returncode == 0 or result.returncode != 1: return result - logger.info("Mounting debugfs to '%s'.", DEBUGFS_PATH) + logger.info("Mounting debugfs to '%s'.", str(DEBUGFS_PATH)) result = common_run_command( - ("sudo", "mount", "-t", "debugfs", "debugfs", DEBUGFS_PATH), + ("sudo", "mount", "-t", "debugfs", "debugfs", str(DEBUGFS_PATH)), ssh_client, ) if result.returncode: - logger.info("Unmounting debugfs from '%s'.", DEBUGFS_PATH) - common_run_command(("sudo", "umount", DEBUGFS_PATH), ssh_client) + logger.info("Unmounting debugfs from '%s'.", str(DEBUGFS_PATH)) + common_run_command(("sudo", "umount", str(DEBUGFS_PATH)), ssh_client) return result @@ -52,7 +53,7 @@ def change_fault_parameters( if result.returncode: return result - result = common_run_command(["ls", DEBUGFS_PATH], ssh_client=client) + result = common_run_command(["ls", str(DEBUGFS_PATH)], ssh_client=client) if result.returncode: logger.error("Failed to list debugfs directory.") return result @@ -63,7 +64,7 @@ def change_fault_parameters( if ("fail" in dir_name or "fault" in dir_name) and dir_name != "fault_around_bytes" ] if not dirs: - logger.warning("No fault-injection debugfs entries found under %s", DEBUGFS_PATH) + logger.warning("No fault-injection debugfs entries found under %s", str(DEBUGFS_PATH)) return result # When fault injection is enabled, times should be set to -1, so that it would run infinitely @@ -71,7 +72,7 @@ def change_fault_parameters( times = -1 if fault_probability > 0 else 1 last_result = result for directory in dirs: - dir_path = DEBUGFS_PATH + "/" + directory + dir_path = DEBUGFS_PATH / directory updates = ( ("interval", fault_interval), ("space", 0), From 6c149aa9278c16825c4d0f18cffd45ed21d0465a Mon Sep 17 00:00:00 2001 From: Artanias <43622365+Artanias@users.noreply.github.com> Date: Sun, 14 Jun 2026 22:16:21 +0300 Subject: [PATCH 11/11] refactor: use library function for calculating subtests timeouts. --- src/imgtests/suites/fault_injection.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/imgtests/suites/fault_injection.py b/src/imgtests/suites/fault_injection.py index 996b5ea9..29b1ce89 100644 --- a/src/imgtests/suites/fault_injection.py +++ b/src/imgtests/suites/fault_injection.py @@ -229,7 +229,11 @@ def _run( return TestResult(status=TestStatus.SKIPPED) stress_ng = StressNg(client) - timeout_suite = (timeout // (len(self.stress_ng_suites) * len(self.fault_probs))) + 1 + timeout_suite = calc_subtest_timeout( + timeout, + len(self.stress_ng_suites) * len(self.fault_probs), + 60, + ) for fault_prob in self.fault_probs: self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) for stress_ng_suite in self.stress_ng_suites: @@ -294,7 +298,11 @@ def _run( return TestResult(status=TestStatus.SKIPPED) perf = Perf(client) - timeout_suite = (timeout // (len(self.perf_suites) * len(self.fault_probs))) + 1 + timeout_suite = calc_subtest_timeout( + timeout, + len(self.perf_suites) * len(self.fault_probs), + 60, + ) for fault_prob in self.fault_probs: self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) for perf_suite in self.perf_suites: @@ -358,7 +366,11 @@ def _run( self.logger.warning("Skipping test due to fault injection is supported on poky.") return TestResult(status=TestStatus.SKIPPED) - timeout_suite = (timeout // (len(self.fio_suites) * len(self.fault_probs))) + 1 + timeout_suite = calc_subtest_timeout( + timeout, + len(self.fio_suites) * len(self.fault_probs), + 60, + ) for fault_prob in self.fault_probs: self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) for fio_suite in self.fio_suites: