diff --git a/src/imgtests/exec/debugfs.py b/src/imgtests/exec/debugfs.py new file mode 100644 index 00000000..f61f1efa --- /dev/null +++ b/src/imgtests/exec/debugfs.py @@ -0,0 +1,94 @@ +import logging +from pathlib import Path +from typing import TYPE_CHECKING, Final + +from imgtests.exec.exec import ExecResult, SSHClient, common_run_command + +if TYPE_CHECKING: + from imgtests.exec.exec import ExecResult, SSHClient + +DEBUGFS_PATH: Final = Path("/sys/kernel/debug") +MAX_FAULT_PROBABILITY: Final = 100 + +logger = logging.getLogger(__name__) + + +def ensure_debugfs(ssh_client: SSHClient | None) -> ExecResult: + """Ensures that debugfs is created and mounted.""" + result = common_run_command(("sudo", "mkdir", "-p", str(DEBUGFS_PATH)), ssh_client) + if result.returncode: + return result + mount_pattern = f"[[:space:]]{DEBUGFS_PATH}[[:space:]]debugfs[[:space:]]" + result = common_run_command( + ("sudo", "grep", "-qs", mount_pattern, "/proc/mounts"), + ssh_client, + ) + if result.returncode == 0 or result.returncode != 1: + return result + logger.info("Mounting debugfs to '%s'.", str(DEBUGFS_PATH)) + result = common_run_command( + ("sudo", "mount", "-t", "debugfs", "debugfs", str(DEBUGFS_PATH)), + ssh_client, + ) + + if result.returncode: + logger.info("Unmounting debugfs from '%s'.", str(DEBUGFS_PATH)) + common_run_command(("sudo", "umount", str(DEBUGFS_PATH)), ssh_client) + return result + + +def validate_fault_probability(fault_prob: int) -> None: + """Checks if fault injection probability is in between borders.""" + if not 0 <= fault_prob <= MAX_FAULT_PROBABILITY: + err_msg = f"fault_probability must be in range 0..{MAX_FAULT_PROBABILITY}." + raise ValueError(err_msg) + + +def change_fault_parameters( + client: SSHClient | None, + fault_probability: int, + fault_interval: int, +) -> ExecResult: + result = ensure_debugfs(client) + if result.returncode: + return result + + result = common_run_command(["ls", str(DEBUGFS_PATH)], ssh_client=client) + if result.returncode: + logger.error("Failed to list debugfs directory.") + return result + + dirs = [ + dir_name + for dir_name in result.stdout.splitlines() + if ("fail" in dir_name or "fault" in dir_name) and dir_name != "fault_around_bytes" + ] + if not dirs: + logger.warning("No fault-injection debugfs entries found under %s", str(DEBUGFS_PATH)) + return result + + # When fault injection is enabled, times should be set to -1, so that it would run infinitely + # When fault injection is disabled, times set to default 1 + times = -1 if fault_probability > 0 else 1 + last_result = result + for directory in dirs: + dir_path = DEBUGFS_PATH / directory + updates = ( + ("interval", fault_interval), + ("space", 0), + ("times", times), + ("probability", fault_probability), + ) + + for parameter, value in updates: + write_cmd = ["echo", f"{value}", ">", f"{dir_path}/{parameter}"] + result = common_run_command(write_cmd, ssh_client=client) + last_result = result + if result.returncode: + logger.error( + "Failed to update fault-injection parameter %s in %s", + parameter, + dir_path, + ) + return result + return last_result diff --git a/src/imgtests/exec/loaders/kirk.py b/src/imgtests/exec/loaders/kirk.py index 7d32d521..8bfaed7c 100644 --- a/src/imgtests/exec/loaders/kirk.py +++ b/src/imgtests/exec/loaders/kirk.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any from imgtests.exec.base_util import GenericUtil +from imgtests.exec.debugfs import ensure_debugfs, validate_fault_probability from imgtests.exec.exec import ExecResult, SSHClient, common_run_command from imgtests.exec.osinfo import get_os_release from imgtests.exec.pkgmgrs.zypper import Zypper @@ -19,8 +20,6 @@ logger = logging.getLogger(__name__) DEFAULT_LTP_RESULTS_DIR = Path("/var/tmp/ltp-results") # noqa: S108 -DEBUGFS_MOUNTPOINT = Path("/sys/kernel/debug") -MAX_FAULT_PROBABILITY = 100 class Kirk(GenericUtil): @@ -90,36 +89,9 @@ def list_suites( return tuple(line.strip() for line in res.stdout.splitlines() if line.strip()) - @staticmethod - def _validate_fault_probability(fault_prob: int) -> None: - """Checks if fault injection probability is in between borders.""" - if not 0 <= fault_prob <= MAX_FAULT_PROBABILITY: - err_msg = f"fault_probability must be in range 0..{MAX_FAULT_PROBABILITY}." - raise ValueError(err_msg) - - def _ensure_debugfs(self) -> ExecResult: - """Ensures that debugfs is created and mounted.""" - debugfs_path = str(DEBUGFS_MOUNTPOINT) - result = common_run_command(("sudo", "mkdir", "-p", debugfs_path), self.ssh_client) - if result.returncode: - return result - mount_pattern = f"[[:space:]]{debugfs_path}[[:space:]]debugfs[[:space:]]" - result = common_run_command( - ("sudo", "grep", "-qs", mount_pattern, "/proc/mounts"), - self.ssh_client, - ) - if result.returncode == 0 or result.returncode != 1: - return result - logger.info("Mounting debugfs to '%s'.", debugfs_path) - result = common_run_command( - ("sudo", "mount", "-t", "debugfs", "debugfs", debugfs_path), - self.ssh_client, - ) - - if result.returncode: - logger.info("Unmounting debugfs from '%s'.", debugfs_path) - common_run_command(("sudo", "umount", debugfs_path), self.ssh_client) - return result + def is_suites_available(self, required_suites: tuple[str, ...]) -> bool: + available_suites = self.list_suites() + return all(suite in available_suites for suite in required_suites) def run( # noqa: PLR0911, PLR0913 self, @@ -152,9 +124,9 @@ def run( # noqa: PLR0911, PLR0913 raise ValueError(scenarios_empty_msg) if fault_prob is not None: - self._validate_fault_probability(fault_prob) + validate_fault_probability(fault_prob) - debugfs_res = self._ensure_debugfs() + debugfs_res = ensure_debugfs(self.ssh_client) if debugfs_res.returncode: return debugfs_res, None diff --git a/src/imgtests/runner.py b/src/imgtests/runner.py index b9806989..ffa359ec 100644 --- a/src/imgtests/runner.py +++ b/src/imgtests/runner.py @@ -172,20 +172,27 @@ def handle_tests( test_instance = test_class else: test_instance = test_class(timeout=self.__test_config.test_duration) - for result in test_instance(self._executor, self._client): - self._database.insert_util_run_result( - experiment_id=experiment_id, - # TODO: fill util_type with the correct value - util_type="loader", - # TODO: fill descriptions and adds into TestResult class - description="", - result=result.metrics, - command=result.command, - started_at=result.started_at, - ended_at=result.ended_at, + try: + for result in test_instance(self._executor, self._client): + self._database.insert_util_run_result( + experiment_id=experiment_id, + # TODO: fill util_type with the correct value + util_type="loader", + # TODO: fill descriptions and adds into TestResult class + description="", + result=result.metrics, + command=result.command, + started_at=result.started_at, + ended_at=result.ended_at, + ) + self.__tests_statuses[result.status] += 1 + self.__tests_cnt += 1 + except Exception: + self._logger.exception( + "Test '%s' failed with exception.", + test_instance.description, ) - self.__tests_statuses[result.status] += 1 - self.__tests_cnt += 1 + self.__tests_statuses[TestStatus.BROKEN] += 1 self._collect_system_errors( experiment_id=experiment_id, since=test_started_at, diff --git a/src/imgtests/suites/fault_injection.py b/src/imgtests/suites/fault_injection.py index a9b8113b..29b1ce89 100644 --- a/src/imgtests/suites/fault_injection.py +++ b/src/imgtests/suites/fault_injection.py @@ -1,8 +1,11 @@ +import logging import random from datetime import UTC, datetime from time import sleep from typing import TYPE_CHECKING +from imgtests.exec.debugfs import change_fault_parameters +from imgtests.exec.exec import ExecResult, SSHClient from imgtests.exec.loaders import Chaosblade, ChaosResponse, Kirk, Perf, StressNg from imgtests.exec.osinfo import get_os_release from imgtests.exec.user_commands import MkDir @@ -17,6 +20,18 @@ from imgtests.exec.exec import ExecResult, SSHClient +logger = logging.getLogger(__name__) + + +class FaultCleanupMixin: + def cleanup(self, client: SSHClient | None, logger: logging.Logger) -> None: + change_fault_parameters(client, 0, 1) + logger.info("Cleaned fault parameters.") + default_cleanup = getattr(super(), "cleanup", None) + if default_cleanup: + default_cleanup(client, logger) + + class FaultInjectionEnduranceTest(AbstractRunnableTimeLimitedTest): def __init__(self, timeout: int, iterations: int = 4) -> None: super().__init__( @@ -41,7 +56,7 @@ def _run( return TestResult(status=TestStatus.SKIPPED) kirk = Kirk(client) - if not is_kirk_suites_available(kirk, self.kirk_suites): + if not kirk.is_suites_available(self.kirk_suites): self.logger.warning("Kirk suite not available for the image with LTP.") return TestResult(status=TestStatus.SKIPPED) @@ -76,7 +91,7 @@ def _run( ) -class FaultInjectionChaosbladeTest(AbstractRunnableTimeLimitedTest): +class FaultInjectionChaosbladeTest(FaultCleanupMixin, AbstractRunnableTimeLimitedTest): def __init__(self, timeout: int) -> None: super().__init__( "Chaosblade test with fault injection.", @@ -84,10 +99,6 @@ def __init__(self, timeout: int) -> None: timeout, ) - @property - def kirk_suites(self) -> tuple[str, str]: - return ("dio", "sched") - @property def chaosblade_suites(self) -> tuple[str, str]: return ("create_disk_exp", "create_cpu_exp") @@ -110,30 +121,22 @@ def _run( mkdir([tmp_dir]) timeout_suite = calc_subtest_timeout( timeout, - len(self.kirk_suites) * len(self.fault_probs), + len(self.chaosblade_suites) * len(self.fault_probs), 60, ) chaosblade = Chaosblade(client) - kirk = Kirk(client) - if not is_kirk_suites_available(kirk, self.kirk_suites): - self.logger.warning("Kirk suite not available for the image with LTP.") - return TestResult(status=TestStatus.SKIPPED) for fault_prob in self.fault_probs: self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) - for kirk_suite, chaosblade_suite in zip( - self.kirk_suites, - self.chaosblade_suites, - strict=True, - ): + for chaosblade_suite in self.chaosblade_suites: started_at = datetime.now(UTC) - kirk_future = executor.submit( - kirk.run, - scenarios=[kirk_suite], - timeout=timeout_suite, - fault_prob=fault_prob, - fault_interval=5, - ) - sleep(1) + result = change_fault_parameters(client, fault_prob, 5) + if result.returncode: + yield TestResult( + command=" ".join(result.cmd), + started_at=started_at, + status=TestStatus.FAILED, + ) + return chaosblade_future = self._create_chaosblade_future( executor, @@ -151,7 +154,9 @@ def _run( chaosblade_result.result, ) while not future.done(): - _, chaosblade_status = chaosblade.get_exp_status(chaosblade_result.result) + _, chaosblade_status = chaosblade.get_exp_status( + chaosblade_result.result, + ) if chaosblade_status.result["Status"] == "Error": status = TestStatus.FAILED break @@ -166,21 +171,6 @@ def _run( status=status, ) - result, metrics_path = kirk_future.result() - if metrics_path: - yield TestResult( - command=" ".join(result.cmd), - metrics=kirk.metrics_to_json(metrics_path), - started_at=started_at, - status=TestStatus.PASSED, - ) - else: - yield TestResult( - command=" ".join(result.cmd), - started_at=started_at, - status=TestStatus.FAILED, - ) - def _create_chaosblade_future( self, executor: ThreadPoolExecutor, @@ -209,7 +199,7 @@ def _create_chaosblade_future( raise ValueError(err_msg) -class FaultInjectionStressNgTest(AbstractRunnableTimeLimitedTest): +class FaultInjectionStressNgTest(FaultCleanupMixin, AbstractRunnableTimeLimitedTest): def __init__(self, timeout: int) -> None: super().__init__( "Stress-ng test with fault injection.", @@ -217,10 +207,6 @@ def __init__(self, timeout: int) -> None: timeout, ) - @property - def kirk_suites(self) -> tuple[str, str]: - return ("dio", "syscalls") - @property def stress_ng_suites(self) -> tuple[dict[str, str | int], dict[str, str | int]]: return ( @@ -242,34 +228,24 @@ def _run( self.logger.warning("Skipping test due to fault injection is supported on poky.") return TestResult(status=TestStatus.SKIPPED) - kirk = Kirk(client) - if not is_kirk_suites_available(kirk, self.kirk_suites): - self.logger.warning("Kirk suite not available for the image with LTP.") - return TestResult(status=TestStatus.SKIPPED) - stress_ng = StressNg(client) timeout_suite = calc_subtest_timeout( timeout, - len(self.kirk_suites) * len(self.fault_probs), + len(self.stress_ng_suites) * len(self.fault_probs), 60, ) - for fault_prob in self.fault_probs: self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) - for kirk_suite, stress_ng_suite in zip( - self.kirk_suites, - self.stress_ng_suites, - strict=True, - ): + for stress_ng_suite in self.stress_ng_suites: started_at = datetime.now(UTC) - kirk_future = executor.submit( - kirk.run, - scenarios=[kirk_suite], - timeout=timeout_suite, - fault_prob=fault_prob, - fault_interval=5, - ) - sleep(1) + result = change_fault_parameters(client, fault_prob, 5) + if result.returncode: + yield TestResult( + command=" ".join(result.cmd), + started_at=started_at, + status=TestStatus.FAILED, + ) + return stress_ng_future = executor.submit( stress_ng.run, @@ -294,23 +270,8 @@ def _run( status=TestStatus.PASSED, ) - result, metrics_path = kirk_future.result() - if metrics_path: - yield TestResult( - command=" ".join(result.cmd), - metrics=kirk.metrics_to_json(metrics_path), - started_at=started_at, - status=TestStatus.PASSED, - ) - else: - yield TestResult( - command=" ".join(result.cmd), - started_at=started_at, - status=TestStatus.FAILED, - ) - -class FaultInjectionPerfTest(AbstractRunnableTimeLimitedTest): +class FaultInjectionPerfTest(FaultCleanupMixin, AbstractRunnableTimeLimitedTest): def __init__(self, timeout: int) -> None: super().__init__( "Perf bench test with fault injection.", @@ -318,10 +279,6 @@ def __init__(self, timeout: int) -> None: timeout, ) - @property - def kirk_suites(self) -> tuple[str, str]: - return ("dio", "syscalls") - @property def perf_suites(self) -> tuple[str, str]: return ("sched", "syscall") @@ -340,34 +297,24 @@ def _run( self.logger.warning("Skipping test due to fault injection is supported on poky.") return TestResult(status=TestStatus.SKIPPED) - kirk = Kirk(client) - if not is_kirk_suites_available(kirk, self.kirk_suites): - self.logger.warning("Kirk suite not available for the image with LTP.") - return TestResult(status=TestStatus.SKIPPED) - perf = Perf(client) timeout_suite = calc_subtest_timeout( timeout, - len(self.kirk_suites) * len(self.fault_probs), + len(self.perf_suites) * len(self.fault_probs), 60, ) - for fault_prob in self.fault_probs: self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) - for kirk_suite, perf_suite in zip( - self.kirk_suites, - self.perf_suites, - strict=True, - ): + for perf_suite in self.perf_suites: started_at = datetime.now(UTC) - kirk_future = executor.submit( - kirk.run, - scenarios=[kirk_suite], - timeout=timeout_suite, - fault_prob=fault_prob, - fault_interval=5, - ) - sleep(1) + result = change_fault_parameters(client, fault_prob, 5) + if result.returncode: + yield TestResult( + command=" ".join(result.cmd), + started_at=started_at, + status=TestStatus.FAILED, + ) + return perf_future = executor.submit( perf.bench, @@ -389,23 +336,8 @@ def _run( status=TestStatus.PASSED, ) - result, metrics_path = kirk_future.result() - if metrics_path: - yield TestResult( - command=" ".join(result.cmd), - metrics=kirk.metrics_to_json(metrics_path), - started_at=started_at, - status=TestStatus.PASSED, - ) - else: - yield TestResult( - command=" ".join(result.cmd), - started_at=started_at, - status=TestStatus.FAILED, - ) - -class FaultInjectionFioTest(AbstractRunnableTimeLimitedTest): +class FaultInjectionFioTest(FaultCleanupMixin, AbstractRunnableTimeLimitedTest): def __init__(self, timeout: int) -> None: super().__init__( "Fio test with fault injection.", @@ -413,10 +345,6 @@ def __init__(self, timeout: int) -> None: timeout, ) - @property - def kirk_suites(self) -> tuple[str, str]: - return ("dio", "syscalls") - @property def fio_suites(self) -> tuple[FioWorkload, ...]: return ( @@ -430,7 +358,7 @@ def fault_probs(self) -> tuple[int, ...]: def _run( self, - executor: ThreadPoolExecutor, + executor: ThreadPoolExecutor, # noqa: ARG002 client: SSHClient | None, timeout: int, ) -> Iterable[TestResult]: @@ -438,33 +366,23 @@ def _run( self.logger.warning("Skipping test due to fault injection is supported on poky.") return TestResult(status=TestStatus.SKIPPED) - kirk = Kirk(client) - if not is_kirk_suites_available(kirk, self.kirk_suites): - self.logger.warning("Kirk suite not available for the image with LTP.") - return TestResult(status=TestStatus.SKIPPED) - timeout_suite = calc_subtest_timeout( timeout, - len(self.kirk_suites) * len(self.fault_probs), + len(self.fio_suites) * len(self.fault_probs), 60, ) - for fault_prob in self.fault_probs: self.logger.info("Run with %d fault_prob and %d timeout", fault_prob, timeout_suite) - for kirk_suite, fio_suite in zip( - self.kirk_suites, - self.fio_suites, - strict=True, - ): + for fio_suite in self.fio_suites: started_at = datetime.now(UTC) - kirk_future = executor.submit( - kirk.run, - scenarios=[kirk_suite], - timeout=timeout_suite, - fault_prob=fault_prob, - fault_interval=5, - ) - sleep(1) + result = change_fault_parameters(client, fault_prob, 5) + if result.returncode: + yield TestResult( + command=" ".join(result.cmd), + started_at=started_at, + status=TestStatus.FAILED, + ) + return fio_config = FioSuiteConfig( suite="fault_injection", @@ -476,27 +394,7 @@ def _run( fio = FioSuite(client, fio_config).run() yield from fio - result, metrics_path = kirk_future.result() - if metrics_path: - yield TestResult( - command=" ".join(result.cmd), - metrics=kirk.metrics_to_json(metrics_path), - started_at=started_at, - status=TestStatus.PASSED, - ) - else: - yield TestResult( - command=" ".join(result.cmd), - started_at=started_at, - status=TestStatus.FAILED, - ) - def is_fault_injection_available(client: SSHClient | None) -> bool: os_id = get_os_release(client).id return not (os_id and os_id != Distro.POKY.value) - - -def is_kirk_suites_available(kirk: Kirk, required_suites: tuple[str, ...]) -> bool: - available_suites = kirk.list_suites() - return all(suite in available_suites for suite in required_suites)