From 61f8a3d9afcc24ca122c93806e35ebc0fe18daca Mon Sep 17 00:00:00 2001 From: Peter Grzybowski Date: Wed, 20 May 2026 09:23:08 +0200 Subject: [PATCH 1/8] chore: debug Ticket: QA-1625 Signed-off-by: Peter Grzybowski --- tests/tests/test_mender_connect.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/tests/test_mender_connect.py b/tests/tests/test_mender_connect.py index 969188071..e0f57d748 100644 --- a/tests/tests/test_mender_connect.py +++ b/tests/tests/test_mender_connect.py @@ -16,6 +16,8 @@ import pytest import time import uuid +import os.path +import os from flaky import flaky @@ -40,6 +42,12 @@ container_factory = factory.get_factory() +def bp(index): + f="/tmp/bp"+str(index) + while not os.path.exists(f): + time.sleep(0.1) + os.remove(f) + class _TestRemoteTerminalBase: @flaky(max_runs=3) From 995c0f78418ee1e6e2e1bb8d04cba06a4c5d5240 Mon Sep 17 00:00:00 2001 From: Peter Grzybowski Date: Wed, 20 May 2026 09:33:46 +0200 Subject: [PATCH 2/8] chore: rm Ticket: QA-1625 Signed-off-by: Peter Grzybowski --- tests/tests/test_00_multi_tenancy.py | 112 -- tests/tests/test_autogenerate_delta.py | 151 -- tests/tests/test_bad_depl_log.py | 215 --- tests/tests/test_basic_integration.py | 461 ----- tests/tests/test_bootstrapping.py | 124 -- tests/tests/test_configuration.py | 197 --- tests/tests/test_db_migration.py | 272 --- tests/tests/test_deployment_aborting.py | 176 -- tests/tests/test_deployment_retry.py | 134 -- tests/tests/test_docker_compose.py | 574 ------ tests/tests/test_fault_tolerance.py | 628 ------- tests/tests/test_filetransfer.py | 781 --------- tests/tests/test_grouping.py | 197 --- tests/tests/test_image_update_failures.py | 116 -- tests/tests/test_inventory.py | 316 ---- tests/tests/test_legacy_golang_update.py | 137 -- tests/tests/test_monitor_client.py | 1920 --------------------- tests/tests/test_mtls.py | 418 ----- tests/tests/test_os_ent_migration.py | 212 --- tests/tests/test_preauth.py | 249 --- tests/tests/test_provides_depends.py | 109 -- tests/tests/test_security.py | 70 - tests/tests/test_signed_image_update.py | 109 -- tests/tests/test_state_scripts.py | 991 ----------- tests/tests/test_tcp_teardown.py | 106 -- tests/tests/test_update_modules.py | 86 - 26 files changed, 8861 deletions(-) delete mode 100644 tests/tests/test_00_multi_tenancy.py delete mode 100644 tests/tests/test_autogenerate_delta.py delete mode 100644 tests/tests/test_bad_depl_log.py delete mode 100644 tests/tests/test_basic_integration.py delete mode 100644 tests/tests/test_bootstrapping.py delete mode 100644 tests/tests/test_configuration.py delete mode 100644 tests/tests/test_db_migration.py delete mode 100644 tests/tests/test_deployment_aborting.py delete mode 100644 tests/tests/test_deployment_retry.py delete mode 100644 tests/tests/test_docker_compose.py delete mode 100644 tests/tests/test_fault_tolerance.py delete mode 100644 tests/tests/test_filetransfer.py delete mode 100644 tests/tests/test_grouping.py delete mode 100644 tests/tests/test_image_update_failures.py delete mode 100644 tests/tests/test_inventory.py delete mode 100644 tests/tests/test_legacy_golang_update.py delete mode 100644 tests/tests/test_monitor_client.py delete mode 100644 tests/tests/test_mtls.py delete mode 100644 tests/tests/test_os_ent_migration.py delete mode 100644 tests/tests/test_preauth.py delete mode 100644 tests/tests/test_provides_depends.py delete mode 100644 tests/tests/test_security.py delete mode 100644 tests/tests/test_signed_image_update.py delete mode 100644 tests/tests/test_state_scripts.py delete mode 100644 tests/tests/test_tcp_teardown.py delete mode 100644 tests/tests/test_update_modules.py diff --git a/tests/tests/test_00_multi_tenancy.py b/tests/tests/test_00_multi_tenancy.py deleted file mode 100644 index 72cac85d5..000000000 --- a/tests/tests/test_00_multi_tenancy.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2022 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import uuid -import time - -from ..common_setup import enterprise_no_client -from ..helpers import Helpers -from ..MenderAPI import auth, devauth, logger, inv -from .common_update import update_image -from .mendertesting import MenderTesting -from testutils.common import new_tenant_client - - -class TestMultiTenancyEnterprise(MenderTesting): - def test_token_validity(self, enterprise_no_client): - """verify that only devices with valid tokens can bootstrap - successfully to a multitenancy setup""" - - wrong_token = "wrong-token" - - auth.reset_auth_token() - uuidv4 = str(uuid.uuid4()) - auth.new_tenant( - "test.mender.io-" + uuidv4, - "some.user+" + uuidv4 + "@example.com", - "hunter2hunter2", - ) - token = auth.current_tenant["tenant_token"] - - # create a new client with an incorrect token set - mender_device = new_tenant_client( - enterprise_no_client, "mender-client", wrong_token - ) - mender_device.ssh_is_opened() - Helpers.check_log_is_unauthenticated( - mender_device, - ) - - for _ in range(5): - time.sleep(5) - devauth.get_devices(expected_devices=0) # make sure device not seen - - # setting the correct token makes the client visible to the backend - mender_device.run( - "sed -i 's/%s/%s/g' /etc/mender/mender.conf" % (wrong_token, token) - ) - mender_device.run("systemctl restart mender-authd") - - devauth.get_devices(expected_devices=1) - - def test_multi_tenancy_deployment( - self, enterprise_no_client, valid_image_with_mender_conf - ): - """Simply make sure we are able to run the multi tenancy setup and - bootstrap 2 different devices to different tenants""" - - auth.reset_auth_token() - - uuidv4 = str(uuid.uuid4()) - users = [ - { - "email": "foo2-%s@example.com" % uuidv4, - "password": "hunter2hunter2", - "username": "test.mender.io-" + uuidv4 + "-foo2", - "container": "mender-client-deployment-1", - }, - { - "email": "bar2-%s@example.com" % uuidv4, - "password": "hunter2hunter2", - "username": "test.mender.io-" + uuidv4 + "-bar2", - "container": "mender-client-deployment-2", - }, - ] - - for user in users: - auth.new_tenant(user["username"], user["email"], user["password"]) - t = auth.current_tenant["tenant_token"] - mender_device = new_tenant_client( - enterprise_no_client, user["container"], t - ) - devauth.accept_devices(1) - assert len(inv.get_devices()) == 1 - - # By default the valid_image has a tenant_token=dummy in the mender.conf - # Therefore, replace the 'mender.conf' in the image with the one from the - # currently running image, which has a valid 'mender.conf'. - mender_conf = mender_device.run("cat /etc/mender/mender.conf") - logger.info( - "Injecting the mender.conf into the update image:\n{conf}\n".format( - conf=mender_conf - ) - ) - update_image_name = valid_image_with_mender_conf(mender_conf) - - host_ip = enterprise_no_client.get_virtual_network_host_ip() - update_image( - mender_device, - host_ip, - install_image=update_image_name, - ) diff --git a/tests/tests/test_autogenerate_delta.py b/tests/tests/test_autogenerate_delta.py deleted file mode 100644 index 509592b03..000000000 --- a/tests/tests/test_autogenerate_delta.py +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright 2023 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import time - -import pytest - -from .. import conftest -from ..common_setup import enterprise_one_rofs_commercial_client_bootstrapped -from .common_update import update_image -from testutils.common import ApiClient -import testutils.api.deployments as deployments - -from ..MenderAPI import ( - image, - DeviceAuthV2, - Deployments, - get_container_manager, -) -from .mendertesting import MenderTesting - - -class BaseTestAutogenerateDelta(MenderTesting): - def do_test_update_with_autogenerated_delta( - self, - env, - valid_image_rofs_commercial_with_mender_conf, - ): - """Upgrade a device with auto-generated delta artifact""" - - mender_device = env.device - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - mender_conf = mender_device.run("cat /etc/mender/mender.conf") - valid_image_rofs = valid_image_rofs_commercial_with_mender_conf(mender_conf) - if valid_image_rofs is None: - pytest.skip( - "Thud branch and older from Yocto does not have R/O rootfs support" - ) - - # Configure delta auto-generation - dc = ApiClient( - deployments.URL_INTERNAL, - host=get_container_manager().get_ip_of_service("mender-deployments")[0] - + ":8080", - schema="http://", - ) - - r = dc.call( - "PUT", - deployments.URL_INTERNAL_CONFIG, - body={ - "delta": { - "enabled": True, - "binary_delta_limits": { - "jobs_in_parallel": {"max": 2}, - "queue_length": {"max": 4}, - }, - "binary_delta": {"timeout": 3600}, - } - }, - path_params={"tenant_id": env.tenant["id"]}, - ) - - assert r.ok, "Failed to set the delta configuration" - - def make_artifact(artifact_file, artifact_id): - return image.make_rootfs_artifact( - valid_image_rofs, - conftest.machine_name, - artifact_id, - artifact_file, - provides={ - "rootfs-image.update-module.mender-binary-delta.mender_update_module": "mender-binary-delta", - "rootfs-image.update-module.mender-binary-delta.version": "mender-image-master", - }, - ) - - host_ip = env.get_virtual_network_host_ip() - update_image( - mender_device, - host_ip, - make_artifact=make_artifact, - devauth=devauth, - deploy=deploy, - ) - - def make_artifact2(artifact_file, artifact_id): - return image.make_rootfs_artifact( - valid_image_rofs, - conftest.machine_name, - artifact_id, - artifact_file, - provides={ - "rootfs-image.update-module.mender-binary-delta.mender_update_module": "mender-binary-delta2", - "rootfs-image.update-module.mender-binary-delta.version": "mender-image-master2", - }, - ) - - # check if client is applying delta - def deployment_triggered_callback(): - isDelta = False - # wait a minute - i = 60 - while i > 0: - output = mender_device.run( - "grep 'xdelta3' /data/mender/deployments.*.log", - warn=True, - ) - if output != "": - isDelta = True - break - i = i - 1 - time.sleep(1) - - assert isDelta, "No sign of applying delta in the deployment logs" - - update_image( - mender_device, - host_ip, - make_artifact=make_artifact2, - devauth=devauth, - deploy=deploy, - deployment_triggered_callback=deployment_triggered_callback, - autogenerate_delta=True, - ) - - -class TestAutogenerateDeltaEnterprise(BaseTestAutogenerateDelta): - @MenderTesting.fast - def test_update_with_autogenerated_delta( - self, - enterprise_one_rofs_commercial_client_bootstrapped, - valid_image_rofs_commercial_with_mender_conf, - ): - self.do_test_update_with_autogenerated_delta( - enterprise_one_rofs_commercial_client_bootstrapped, - valid_image_rofs_commercial_with_mender_conf, - ) diff --git a/tests/tests/test_bad_depl_log.py b/tests/tests/test_bad_depl_log.py deleted file mode 100644 index ec1dec26c..000000000 --- a/tests/tests/test_bad_depl_log.py +++ /dev/null @@ -1,215 +0,0 @@ -# Copyright 2026 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import shutil -import os -import tempfile -import subprocess - -import pytest - -from .. import conftest -from .common_update import common_update_procedure -from ..helpers import Helpers -from ..MenderAPI import DeviceAuthV2, Deployments, logger, image -from .mendertesting import MenderTesting -from testutils.infra.device import MenderDeviceGroup - -from ..common_setup import ( - class_persistent_standard_setup_one_client_bootstrapped, - class_persistent_enterprise_one_client_bootstrapped, -) -from .test_state_scripts import ( - class_persistent_setup_client_state_scripts_update_module, - class_persistent_enterprise_setup_client_state_scripts_update_module, -) - - -class BaseTestCorruptDeploymentLog(MenderTesting): - def do_test_corrupt_deployment_log(self, env): - """Test that corrupted deployment log is sanitized and successfully - uploaded to the server.""" - - mender_device = env.device - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - work_dir = "test_corrupt_depl_log.%s" % mender_device.host_string - deployment_id = None - try: - artifact_script_dir = os.path.join(work_dir, "artifact-scripts") - os.makedirs(artifact_script_dir) - with open( - os.path.join( - artifact_script_dir, "ArtifactInstall_Leave_01_corrupt-depl-log" - ), - "w", - ) as fd: - fd.write("""#!/bin/sh - log_file=$(ls /var/lib/mender/deployments.0000.*.log) - echo break >> $log_file - sync $log_file - exit 1 - """) - - # Callback for our custom artifact maker - def make_artifact(filename, artifact_name): - return image.make_module_artifact( - "module-state-scripts-test", - conftest.machine_name, - artifact_name, - filename, - scripts=[artifact_script_dir], - ) - - # Now create the artifact, and make the deployment. - device_id = Helpers.ip_to_device_id_map( - MenderDeviceGroup([mender_device.host_string]), - devauth=devauth, - )[mender_device.host_string] - deployment_id = common_update_procedure( - verify_status=False, - devices=[device_id], - scripts=[artifact_script_dir], - make_artifact=make_artifact, - devauth=devauth, - deploy=deploy, - )[0] - - deploy.check_expected_statistics(deployment_id, "failure", 1) - logs = deploy.get_logs(device_id, deployment_id) - assert "(THE ORIGINAL LOGS CONTAINED INVALID ENTRIES)" in logs - except: - output = mender_device.run( - "cat /data/mender/deployment*.log", warn_only=True - ) - logger.info(output) - raise - finally: - shutil.rmtree(work_dir, ignore_errors=True) - if deployment_id: - try: - deploy.abort(deployment_id) - except: - pass - - def do_test_large_deployment_log(self, env): - """Test that large deployment log is trimmed and successfully - uploaded to the server.""" - - mender_device = env.device - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - work_dir = tempfile.mkdtemp( - suffix=f"test_large_depl_log.{mender_device.host_string}" - ) - deployment_id = None - try: - artifact_script_dir = os.path.join(work_dir, "artifact-scripts") - os.makedirs(artifact_script_dir) - with open( - os.path.join( - artifact_script_dir, "ArtifactInstall_Leave_01_enlarge-depl-log" - ), - "w", - ) as fd: - fd.write("""#!/bin/sh - log_file=$(ls /var/lib/mender/deployments.0000.*.log) - for i in `seq 1 100`; do - for j in `seq 1 500`; do - echo '{"timestamp": "1970-01-01T00:00:00.000000000Z", "level": "ERROR", "message": "some useless log message here"}' >> $log_file - done - done - sync $log_file - exit 1 - """) - - # Callback for our custom artifact maker - def make_artifact(filename, artifact_name): - return image.make_module_artifact( - "module-state-scripts-test", - conftest.machine_name, - artifact_name, - filename, - scripts=[artifact_script_dir], - ) - - # Now create the artifact, and make the deployment. - device_id = Helpers.ip_to_device_id_map( - MenderDeviceGroup([mender_device.host_string]), - devauth=devauth, - )[mender_device.host_string] - deployment_id = common_update_procedure( - verify_status=False, - devices=[device_id], - scripts=[artifact_script_dir], - make_artifact=make_artifact, - devauth=devauth, - deploy=deploy, - )[0] - - deploy.check_expected_statistics(deployment_id, "failure", 1) - logs = deploy.get_logs(device_id, deployment_id, n_tries=5) - assert "(THE ORIGINAL LOGS WERE TOO BIG" in logs - except: - output = mender_device.run( - "cat /data/mender/deployment*.log | grep -v 'some useless log message here'", - warn_only=True, - ) - logger.info(output) - raise - finally: - shutil.rmtree(work_dir, ignore_errors=True) - if deployment_id: - try: - deploy.abort(deployment_id) - except: - pass - - -class TestStateScriptsOpenSource(BaseTestCorruptDeploymentLog): - def test_corrupt_deployment_log( - self, - class_persistent_setup_client_state_scripts_update_module, - ): - self.do_test_corrupt_deployment_log( - class_persistent_setup_client_state_scripts_update_module, - ) - - def test_large_deployment_log( - self, - class_persistent_setup_client_state_scripts_update_module, - ): - self.do_test_large_deployment_log( - class_persistent_setup_client_state_scripts_update_module, - ) - - -class TestStateScriptsEnterprise(BaseTestCorruptDeploymentLog): - def test_corrupt_deployment_log( - self, - class_persistent_enterprise_setup_client_state_scripts_update_module, - ): - self.do_test_corrupt_deployment_log( - class_persistent_enterprise_setup_client_state_scripts_update_module, - ) - - def test_large_deployment_log( - self, - class_persistent_enterprise_setup_client_state_scripts_update_module, - ): - self.do_test_large_deployment_log( - class_persistent_enterprise_setup_client_state_scripts_update_module, - ) diff --git a/tests/tests/test_basic_integration.py b/tests/tests/test_basic_integration.py deleted file mode 100644 index f206eb794..000000000 --- a/tests/tests/test_basic_integration.py +++ /dev/null @@ -1,461 +0,0 @@ -# Copyright 2023 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import pytest -import shutil -import time - -from ..common_setup import ( - standard_setup_one_rofs_client_bootstrapped, - standard_setup_with_short_lived_token, - setup_failover, - standard_setup_one_client_bootstrapped, - enterprise_one_client_bootstrapped, - enterprise_one_rofs_client_bootstrapped, - enterprise_with_short_lived_token, -) -from .common_update import update_image, update_image_failed -from ..MenderAPI import ( - image, - logger, - devauth, - DeviceAuthV2, - Deployments, - Inventory, - get_container_manager, -) -from .mendertesting import MenderTesting -from ..helpers import Helpers - - -class DeviceAuthFailover(DeviceAuthV2): - def __init__(self, devauth): - self.auth = devauth.auth - - def get_devauth_base_path(self): - return "https://%s/api/management/v2/devauth/" % ( - get_container_manager().get_ip_of_service("mender-api-gateway-2")[0] - ) - - -class BaseTestBasicIntegration(MenderTesting): - def do_test_double_update_rofs(self, env, valid_image_rofs_with_mender_conf): - """Upgrade a device with two consecutive R/O images using different compression algorithms""" - - mender_device = env.device - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - mender_conf = mender_device.run("cat /etc/mender/mender.conf") - valid_image_rofs = valid_image_rofs_with_mender_conf(mender_conf) - if valid_image_rofs is None: - pytest.skip( - "Thud branch and older from Yocto does not have R/O rootfs support" - ) - - # Verify that partition is read-only as expected - mender_device.run("mount | fgrep 'on / ' | fgrep '(ro,'") - - host_ip = env.get_virtual_network_host_ip() - update_image( - mender_device, - host_ip, - install_image=valid_image_rofs, - compression_type="gzip", - devauth=devauth, - deploy=deploy, - ) - mender_device.run("mount | fgrep 'on / ' | fgrep '(ro,'") - - update_image( - mender_device, - host_ip, - install_image=valid_image_rofs, - compression_type="lzma", - devauth=devauth, - deploy=deploy, - ) - mender_device.run("mount | fgrep 'on / ' | fgrep '(ro,'") - - def do_test_update_jwt_expired(self, env, valid_image_with_mender_conf): - """Update a device with a short lived JWT token""" - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - mender_conf = env.device.run("cat /etc/mender/mender.conf") - update_image( - env.device, - env.get_virtual_network_host_ip(), - install_image=valid_image_with_mender_conf(mender_conf), - devauth=devauth, - deploy=deploy, - ) - - def do_test_failed_updated_and_valid_update( - self, env, valid_image_with_mender_conf, broken_update_image - ): - """Upload a device with a broken image, followed by a valid image""" - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - mender_device = env.device - host_ip = env.get_virtual_network_host_ip() - - update_image_failed(mender_device, host_ip, devauth=devauth, deploy=deploy) - mender_conf = mender_device.run("cat /etc/mender/mender.conf") - update_image( - mender_device, - host_ip, - install_image=valid_image_with_mender_conf(mender_conf), - devauth=devauth, - deploy=deploy, - ) - - def do_test_update_no_compression(self, env, valid_image_with_mender_conf): - """Uploads an uncompressed artifact, and runs the whole update process.""" - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - mender_device = env.device - mender_conf = mender_device.run("cat /etc/mender/mender.conf") - - update_image( - env.device, - env.get_virtual_network_host_ip(), - install_image=valid_image_with_mender_conf(mender_conf), - compression_type="none", - devauth=devauth, - deploy=deploy, - ) - - def do_test_update_zstd_compression(self, env, valid_image_with_mender_conf): - """Uploads a zstd-compressed artifact, and runs the whole update process.""" - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - mender_device = env.device - mender_conf = mender_device.run("cat /etc/mender/mender.conf") - - update_image( - env.device, - env.get_virtual_network_host_ip(), - install_image=valid_image_with_mender_conf(mender_conf), - compression_type="zstd_best", - devauth=devauth, - deploy=deploy, - ) - - def do_test_forced_update_check_from_client( - self, env, valid_image_with_mender_conf - ): - """Upload a device with a broken image, followed by a valid image""" - - mender_device = env.device - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - # Give the image a really large wait interval. - sedcmd = "sed -i.bak 's/%s/%s/' /etc/mender/mender.conf" % ( - r"\(.*PollInter.*:\)\( *[0-9]*\)", - "\\1 1800", - ) - mender_device.run(sedcmd) - mender_device.run("systemctl restart mender-updated") - - def deployment_callback(): - logger.info("Running pre deployment callback function") - wait_count = 0 - # Match the log template six times to make sure the client is truly sleeping. - catcmd = "journalctl --unit mender-updated --output cat" - template = mender_device.run(catcmd) - while True: - logger.info("sleeping...") - logger.info("wait_count: %d" % wait_count) - time.sleep(10) - out = mender_device.run(catcmd) - if out == template: - wait_count += 1 - # Only return if the client has been idling in check-wait for a minute. - if wait_count == 6: - return - continue - # Update the matching template - template = mender_device.run(catcmd) - wait_count = 0 - - def deployment_triggered_callback(): - mender_device.run("mender-update check-update") - logger.info("mender client has forced an update check") - - mender_conf = mender_device.run("cat /etc/mender/mender.conf") - update_image( - mender_device, - env.get_virtual_network_host_ip(), - install_image=valid_image_with_mender_conf(mender_conf), - pre_deployment_callback=deployment_callback, - deployment_triggered_callback=deployment_triggered_callback, - devauth=devauth, - deploy=deploy, - ) - - def do_test_forced_inventory_update_from_client(self, env): - """Forces an inventory update from an idling client.""" - - mender_device = env.device - inv = Inventory(env.auth) - - # Give the image a really large wait interval. - sedcmd = "sed -i.bak 's/%s/%s/' /etc/mender/mender.conf" % ( - r"\(.*PollInter.*:\)\( *[0-9]*\)", - "\\1 1800", - ) - mender_device.run(sedcmd) - mender_device.run("systemctl restart mender-updated") - - logger.info("Running pre deployment callback function") - wait_count = 0 - # Match the log template six times to make sure the client is truly sleeping. - catcmd = "journalctl --unit mender-updated --output cat" - template = mender_device.run(catcmd) - while True: - logger.info("sleeping...") - logger.info("wait_count: %d" % wait_count) - time.sleep(10) - out = mender_device.run(catcmd) - if out == template: - wait_count += 1 - # Only return if the client has been idling in check-wait for a minute. - if wait_count == 6: - break - continue - # Update the matching template. - template = mender_device.run(catcmd) - wait_count = 0 - - # Create some new inventory data from an inventory script. - mender_device.run( - "cd /usr/share/mender/inventory && echo '#!/bin/sh\necho host=foobar' > mender-inventory-test && chmod +x mender-inventory-test" - ) - - # Now that the client has settled into the wait-state, run the command, and check if it does indeed exit the wait state, - # and send inventory. - mender_device.run("mender-update send-inventory") - logger.info("mender client has forced an inventory update") - - for i in range(10): - # Check that the updated inventory value is now present. - invJSON = inv.get_devices() - for element in invJSON[0]["attributes"]: - if element["name"] == "host" and element["value"] == "foobar": - return - time.sleep(10) - - pytest.fail("The inventory was not updated") - - -class TestBasicIntegrationOpenSource(BaseTestBasicIntegration): - @MenderTesting.fast - def test_update_failover_server(self, setup_failover, valid_image): - """ - Client is initially set up against server A, and then receives an update - containing a multi-server configuration, with server B as primary and A - secondary. Server B does not, however, expect any clients and will trigger - "failover" to server A. - To create the necessary configuration I use a state script to modify the - /etc/mender/mender.conf - """ - - mender_device = setup_failover.device - - tmp_image = valid_image.split(".")[0] + "-failover-image.ext4" - try: - logger.info("Creating failover sample image.") - shutil.copy(valid_image, tmp_image) - conf = image.get_mender_conf(tmp_image) - - if conf is None: - raise SystemExit("Could not retrieve mender.conf") - - conf["Servers"] = [ - {"ServerURL": "https://failover.docker.mender.io"}, - {"ServerURL": conf["ServerURL"]}, - ] - conf.pop("ServerURL") - image.replace_mender_conf(tmp_image, conf) - - host_ip = setup_failover.get_virtual_network_host_ip() - update_image(mender_device, host_ip, install_image=tmp_image) - - # Now try to decommission the device from server A and have it - # accepted in server B. - devices = devauth.get_devices_status() - assert len(devices) == 1 - devauth.decommission(devices[0]["id"]) - - # Journalctl has resolution of one second, so wait one second to - # avoid race conditions when detecting below. - time.sleep(1) - date = mender_device.run('date "+%Y-%m-%d %H:%M:%S"').strip() - - devauth_failover = DeviceAuthFailover(devauth) - - devices = devauth_failover.get_devices_status(status="pending") - assert len(devices) == 1 - - devauth_failover.accept_devices(1) - Helpers.check_log_is_authenticated(mender_device, date) - - # Old server should have no devices now. - devices = devauth.get_devices_status(status="accepted") - assert len(devices) == 0 - - finally: - os.remove(tmp_image) - - @MenderTesting.fast - def test_double_update_rofs( - self, - standard_setup_one_rofs_client_bootstrapped, - valid_image_rofs_with_mender_conf, - ): - self.do_test_double_update_rofs( - standard_setup_one_rofs_client_bootstrapped, - valid_image_rofs_with_mender_conf, - ) - - @MenderTesting.fast - def test_update_jwt_expired( - self, standard_setup_with_short_lived_token, valid_image_with_mender_conf - ): - self.do_test_update_jwt_expired( - standard_setup_with_short_lived_token, valid_image_with_mender_conf - ) - - @MenderTesting.fast - def test_failed_updated_and_valid_update( - self, - standard_setup_one_client_bootstrapped, - valid_image_with_mender_conf, - broken_update_image, - ): - self.do_test_failed_updated_and_valid_update( - standard_setup_one_client_bootstrapped, - valid_image_with_mender_conf, - broken_update_image, - ) - - def test_update_no_compression( - self, - standard_setup_one_client_bootstrapped, - valid_image_with_mender_conf, - ): - self.do_test_update_no_compression( - standard_setup_one_client_bootstrapped, - valid_image_with_mender_conf, - ) - - def test_update_zstd_compression( - self, - standard_setup_one_client_bootstrapped, - valid_image_with_mender_conf, - ): - self.do_test_update_zstd_compression( - standard_setup_one_client_bootstrapped, - valid_image_with_mender_conf, - ) - - def test_forced_update_check_from_client( - self, standard_setup_one_client_bootstrapped, valid_image_with_mender_conf - ): - self.do_test_forced_update_check_from_client( - standard_setup_one_client_bootstrapped, valid_image_with_mender_conf - ) - - @pytest.mark.timeout(1000) - def test_forced_inventory_update_from_client( - self, standard_setup_one_client_bootstrapped - ): - self.do_test_forced_inventory_update_from_client( - standard_setup_one_client_bootstrapped - ) - - -class TestBasicIntegrationEnterprise(BaseTestBasicIntegration): - @MenderTesting.fast - def test_double_update_rofs( - self, - enterprise_one_rofs_client_bootstrapped, - valid_image_rofs_with_mender_conf, - ): - self.do_test_double_update_rofs( - enterprise_one_rofs_client_bootstrapped, - valid_image_rofs_with_mender_conf, - ) - - @MenderTesting.fast - def test_update_jwt_expired( - self, enterprise_with_short_lived_token, valid_image_with_mender_conf - ): - self.do_test_update_jwt_expired( - enterprise_with_short_lived_token, valid_image_with_mender_conf - ) - - @MenderTesting.fast - def test_failed_updated_and_valid_update( - self, - enterprise_one_client_bootstrapped, - valid_image_with_mender_conf, - broken_update_image, - ): - self.do_test_failed_updated_and_valid_update( - enterprise_one_client_bootstrapped, - valid_image_with_mender_conf, - broken_update_image, - ) - - def test_update_no_compression( - self, - enterprise_one_client_bootstrapped, - valid_image_with_mender_conf, - ): - self.do_test_update_no_compression( - enterprise_one_client_bootstrapped, - valid_image_with_mender_conf, - ) - - def test_update_zstd_compression( - self, - enterprise_one_client_bootstrapped, - valid_image_with_mender_conf, - ): - self.do_test_update_zstd_compression( - enterprise_one_client_bootstrapped, - valid_image_with_mender_conf, - ) - - def test_forced_update_check_from_client( - self, enterprise_one_client_bootstrapped, valid_image_with_mender_conf - ): - self.do_test_forced_update_check_from_client( - enterprise_one_client_bootstrapped, valid_image_with_mender_conf - ) - - @pytest.mark.timeout(1000) - def test_forced_inventory_update_from_client( - self, enterprise_one_client_bootstrapped - ): - self.do_test_forced_inventory_update_from_client( - enterprise_one_client_bootstrapped - ) diff --git a/tests/tests/test_bootstrapping.py b/tests/tests/test_bootstrapping.py deleted file mode 100644 index c47527195..000000000 --- a/tests/tests/test_bootstrapping.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright 2023 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import time - -import pytest - -from ..common_setup import ( - standard_setup_one_client, - standard_setup_one_client_bootstrapped, - enterprise_one_client, - enterprise_one_client_bootstrapped, -) -from .common_update import common_update_procedure -from ..MenderAPI import DeviceAuthV2, Deployments, logger -from ..helpers import Helpers -from .mendertesting import MenderTesting - - -class BaseTestBootstrapping(MenderTesting): - def do_test_bootstrap(self, env): - """Simply make sure we are able to bootstrap a device""" - - mender_device = env.device - - devauth = DeviceAuthV2(env.auth) - devauth.check_expected_status("pending", 1) - - # iterate over devices and accept them - for d in devauth.get_devices(): - devauth.set_device_auth_set_status( - d["id"], d["auth_sets"][0]["id"], "accepted" - ) - logger.info("Accepting DeviceID: %s" % d["id"]) - - # make sure all devices are accepted - devauth.check_expected_status("accepted", 1) - Helpers.check_log_is_authenticated(mender_device) - - # print all device ids - for device in devauth.get_devices_status("accepted"): - logger.info("Accepted DeviceID: %s" % device["id"]) - - def do_test_reject_bootstrap(self, env, valid_image): - """Make sure a rejected device does not perform an upgrade, and that it gets it's auth token removed""" - - mender_device = env.device - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - reject_time = time.time() - - # iterate over devices and reject them - for device in devauth.get_devices(): - devauth.set_device_auth_set_status( - device["id"], device["auth_sets"][0]["id"], "rejected" - ) - logger.info("Rejecting DeviceID: %s" % device["id"]) - - devauth.check_expected_status("rejected", 1) - - host_ip = env.get_virtual_network_host_ip() - with mender_device.get_reboot_detector(host_ip) as reboot: - try: - common_update_procedure( - install_image=valid_image, devauth=devauth, deploy=deploy - ) - except AssertionError: - logger.info("Failed to deploy upgrade to rejected device.") - reboot.verify_reboot_not_performed() - - else: - # use assert to fail, so we can get backend logs - pytest.fail("no error while trying to deploy to rejected device") - - # Check from client side - Helpers.check_log_is_unauthenticated( - mender_device, - time.strftime("%Y-%m-%d %H:%M:%S UTC", time.gmtime(reject_time)), - ) - - # Restart client to force log reset. - mender_device.run("systemctl restart mender-updated") - - # Check that we can accept again the device from the server - devauth.accept_devices(1) - - # Check from client side that it can be authorized - Helpers.check_log_is_authenticated(mender_device) - - -class TestBootstrappingOpenSource(BaseTestBootstrapping): - @MenderTesting.fast - def test_bootstrap(self, standard_setup_one_client): - self.do_test_bootstrap(standard_setup_one_client) - - @MenderTesting.slow - def test_reject_bootstrap( - self, standard_setup_one_client_bootstrapped, valid_image - ): - self.do_test_reject_bootstrap( - standard_setup_one_client_bootstrapped, valid_image - ) - - -class TestBootstrappingEnterprise(BaseTestBootstrapping): - @MenderTesting.fast - def test_bootstrap(self, enterprise_one_client): - self.do_test_bootstrap(enterprise_one_client) - - @MenderTesting.slow - def test_reject_bootstrap(self, enterprise_one_client_bootstrapped, valid_image): - self.do_test_reject_bootstrap(enterprise_one_client_bootstrapped, valid_image) diff --git a/tests/tests/test_configuration.py b/tests/tests/test_configuration.py deleted file mode 100644 index 29fca3504..000000000 --- a/tests/tests/test_configuration.py +++ /dev/null @@ -1,197 +0,0 @@ -# Copyright 2022 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import json -import pytest -import redo -import time -import uuid - -from testutils.infra.cli import CliTenantadm -from testutils.common import Tenant, User, update_tenant, new_tenant_client - -from ..common_setup import standard_setup_one_client, enterprise_no_client -from ..MenderAPI import ( - Authentication, - DeviceAuthV2, - authentication, - devauth, - get_container_manager, - logger, -) -from ..MenderAPI.requests_helpers import requests_retry -from .mendertesting import MenderTesting -from .common_connect import wait_for_connect - - -@pytest.mark.usefixtures("standard_setup_one_client") -class TestConfiguration(MenderTesting): - """Tests the configuration deployment functionality""" - - def test_configuration(self, standard_setup_one_client): - """Tests the deployment and reporting of the configuration - - The tests set the configuration of a device and verifies the new - configuration is reported back to the back-end. - """ - # accept the device - devauth.accept_devices(1) - - # list of devices - devices = list( - set([device["id"] for device in devauth.get_devices_status("accepted")]) - ) - assert 1 == len(devices) - - auth = authentication.Authentication() - - wait_for_connect(auth, devices[0]) - - # set and verify the device's configuration - # retry to skip possible race conditions between update poll and update trigger - for _ in redo.retrier(attempts=3, sleeptime=1): - set_and_verify_config({"key": "value"}, devices[0], auth.get_auth_token()) - - forced = was_update_forced(standard_setup_one_client.device) - if forced: - return - - assert False, "the update check was never triggered" - - -@pytest.mark.usefixtures("enterprise_no_client") -class TestConfigurationEnterprise(MenderTesting): - """Tests the configuration deployment functionality in the enterprise setup""" - - def test_configuration(self, enterprise_no_client): - """Tests the deployment and reporting of the configuration - - The tests set the configuration of a device and verifies the new - configuration is reported back to the back-end. - """ - - env = enterprise_no_client - - # Create an enterprise plan tenant - uuidv4 = str(uuid.uuid4()) - tname = "test.mender.io-{}".format(uuidv4) - email = "some.user+{}@example.com".format(uuidv4) - u = User("", email, "whatsupdoc") - cli = CliTenantadm(containers_namespace=env.name) - tid = cli.create_org(tname, u.name, u.pwd, plan="enterprise") - - # what we really need is "configure" - # but for trigger tests we're also checking device avail. in "deviceconnect" - # so add "troubleshoot" as well - update_tenant( - tid, - addons=["configure", "troubleshoot"], - container_manager=get_container_manager(), - ) - - tenant = cli.get_tenant(tid) - tenant = json.loads(tenant) - ttoken = tenant["tenant_token"] - logger.info(f"tenant json: {tenant}") - tenant = Tenant("tenant", tid, ttoken) - tenant.users.append(u) - - # And authorize the user to the tenant account - auth = Authentication(name="enterprise-tenant", username=u.name, password=u.pwd) - auth.create_org = False - auth.reset_auth_token() - devauth_tenant = DeviceAuthV2(auth) - - # Add a client to the tenant - mender_device = new_tenant_client( - enterprise_no_client, "mender-client", tenant.tenant_token - ) - mender_device.ssh_is_opened() - - devauth_tenant.accept_devices(1) - - # list of devices - devices = list( - set( - [ - device["id"] - for device in devauth_tenant.get_devices_status("accepted") - ] - ) - ) - assert 1 == len(devices) - - wait_for_connect(auth, devices[0]) - - # set and verify the device's configuration - # retry to skip possible race conditions between update poll and update trigger - for _ in redo.retrier(attempts=3, sleeptime=1): - set_and_verify_config({"key": "value"}, devices[0], auth.get_auth_token()) - - forced = was_update_forced(mender_device) - if forced: - return - - assert False, "the update check was never triggered" - - -def was_update_forced(mender_device): - """Check that the update was triggered by update-check""" - - out = mender_device.run("journalctl --unit mender-updated --full") - if "SIGUSR1 received, triggering deployments check" in out: - return True - elif "/usr/share/mender/modules/v3/mender-configure" in out: - # deployment was processed too quickly - return False - else: - raise RuntimeError( - "fatal: no expected evidence of an update was found in device logs" - ) - - -def set_and_verify_config(config, devid, authtoken): - """Deploy a configuration and assert it was reported back""" - - configuration_url = ( - "https://%s/api/management/v1/deviceconfig/configurations/device/%s" - % (get_container_manager().get_mender_gateway(), devid) - ) - r = requests_retry().put( - configuration_url, - verify=False, - headers=authtoken, - json=config, - ) - - # deploy the configurations - r = requests_retry().post( - configuration_url + "/deploy", - verify=False, - headers=authtoken, - json={"retries": 0}, - ) - - # loop and verify the reported configuration - reported = None - for i in range(180): - r = requests_retry().get(configuration_url, verify=False, headers=authtoken) - assert r.status_code == 200 - reported = r.json().get("reported") - if reported == config: - break - time.sleep(1) - - assert config == reported diff --git a/tests/tests/test_db_migration.py b/tests/tests/test_db_migration.py deleted file mode 100644 index 004a4c14b..000000000 --- a/tests/tests/test_db_migration.py +++ /dev/null @@ -1,272 +0,0 @@ -# Copyright 2022 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import pytest -import shutil -import tempfile - -from ..common_setup import ( - setup_with_legacy_v1_client, - enterprise_with_legacy_v1_client, -) -from .common_update import update_image, common_update_procedure -from ..MenderAPI import DeviceAuthV2, Deployments, logger -from .mendertesting import MenderTesting - - -class BaseTestDBMigration(MenderTesting): - def ensure_persistent_conf_script(self, dir): - # Because older versions of Yocto branches did not split mender.conf - # into /etc/mender/mender.conf and /data/mender/mender.conf, we need to - # provide the content of the second file ourselves. - name = os.path.join(dir, "ArtifactInstall_Enter_00_ensure_persistent_conf") - with open(name, "w") as fd: - fd.write("""#!/bin/sh - -set -e - -if ! [ -f /data/mender/mender.conf ]; then - ( - echo '{' - grep RootfsPart /etc/mender/mender.conf |sed -e '${s/,$//}' - echo '}' - ) > /data/mender/mender.conf -fi -exit 0 -""") - return name - - def generate_storage_device_state_scripts(self, dir): - # Older versions of our mender-client-qemu image had /dev/hda as their - # storage, in kirkstone, this switched to /dev/sda, so we need to make - # this conversion both when upgrading, and rolling back. - - content = """#!/bin/sh - -detect_image_type_on_passive() { - # Sanity check that this is a Poky build. - if ! grep Poky "$1/etc/os-release" > /dev/null; then - echo "This test is not adapted to non-Poky builds!" 1>&2 - exit 127 - fi - - eval "$(grep '^VERSION_ID=' "$1/etc/os-release")" - printf '%s\n%s\n' "$VERSION_ID" 3.5 > /tmp/versions.txt - # If the smallest is 3.5, it means VERSION_ID is higher or equal, which - # means kirkstone or higher. - if [ "$(sort -V /tmp/versions.txt | head -n 1)" = "3.5" ]; then - echo "/dev/sda" - else - echo "/dev/hda" - fi -} - -if mount | grep "2 on / "; then - eval $(printf PASSIVE=%s /dev/[hs]da3) -else - eval $(printf PASSIVE=%s /dev/[hs]da2) -fi - -mount "$PASSIVE" /mnt -DEV="$(detect_image_type_on_passive /mnt)" -umount /mnt - -for file in /data/mender/mender.conf $(find /boot/efi/ -name grub.cfg); do - sed -i -e "s,/dev/[hs]da,$DEV,g" "$file" -done -""" - - scripts = [ - os.path.join(dir, "ArtifactInstall_Leave_10_storage_device"), - os.path.join(dir, "ArtifactRollback_Leave_10_storage_device"), - ] - for script in scripts: - with open(script, "w") as fd: - fd.write(content) - return scripts - - def do_test_migrate_from_legacy_mender_v1_failure( - self, env, valid_image_with_mender_conf - ): - """ - Start a legacy client (1.7.0) first and update it to the new one. - - The test starts a setup with the 1.7.0 client and then updates it to - the current version. The update is failing first (due to failure - returned inside the artifact commit enter state script). - After the failed first update we are updating cient (1.7.0) again, - and this time the update should succeed. - """ - - mender_device = env.device - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - dirpath = tempfile.mkdtemp() - script_content = "#!/bin/sh\nexit 1\n" - with open(os.path.join(dirpath, "ArtifactCommit_Enter_01"), "w") as fd: - fd.write(script_content) - - active_part = mender_device.get_active_partition() - - ensure_persistent_conf = self.ensure_persistent_conf_script(dirpath) - storage_device_state_scripts = self.generate_storage_device_state_scripts( - dirpath - ) - - mender_conf = mender_device.run("cat /etc/mender/mender.conf") - mender_conf_json = json.loads(mender_conf) - # Delete these, we want the persistent_conf above to take effect. - del mender_conf_json["RootfsPartA"] - del mender_conf_json["RootfsPartB"] - valid_image = valid_image_with_mender_conf(json.dumps(mender_conf_json)) - - # first start with the failed update - host_ip = env.get_virtual_network_host_ip() - with mender_device.get_reboot_detector(host_ip) as reboot: - deployment_id, _ = common_update_procedure( - valid_image, - scripts=[ - ensure_persistent_conf, - os.path.join(dirpath, "ArtifactCommit_Enter_01"), - ] - + storage_device_state_scripts, - version=2, - devauth=devauth, - deploy=deploy, - ) - - logger.info("waiting for system to reboot twice") - reboot.verify_reboot_performed(number_of_reboots=2) - - assert mender_device.get_active_partition() == active_part - deploy.check_expected_statistics(deployment_id, "failure", 1) - - # do the next update, this time successful - update_image( - mender_device, - host_ip, - scripts=[ensure_persistent_conf] + storage_device_state_scripts, - install_image=valid_image, - version=2, - devauth=devauth, - deploy=deploy, - ) - - def do_test_migrate_from_legacy_mender_v1_success( - self, env, valid_image_with_mender_conf - ): - """ - Start a legacy client (1.7.0) first and update it to the new one. - - The test starts a setup with the 1.7.0 client and then updates it to - the current version. After the first successful update, we are updating - the client for the second time, to make sure the DB migration has not left - any traces in the database that are causing issues. - """ - - mender_device = env.device - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - tmpdir = tempfile.mkdtemp() - test_log = "/var/lib/mender/migration_state_scripts.log" - try: - ensure_persistent_conf = self.ensure_persistent_conf_script(tmpdir) - storage_device_state_scripts = self.generate_storage_device_state_scripts( - tmpdir - ) - - # Test that state scripts are also executed correctly. - scripts = ["ArtifactInstall_Enter_00", "ArtifactCommit_Enter_00"] - scripts_paths = [] - for script in scripts: - script_path = os.path.join(tmpdir, script) - scripts_paths += [script_path] - with open(script_path, "w") as fd: - fd.write("#!/bin/sh\necho $(basename $0) >> %s\n" % test_log) - - mender_conf = mender_device.run("cat /etc/mender/mender.conf") - mender_conf_json = json.loads(mender_conf) - # Delete these, we want the persistent_conf above to take effect. - del mender_conf_json["RootfsPartA"] - del mender_conf_json["RootfsPartB"] - valid_image = valid_image_with_mender_conf(json.dumps(mender_conf_json)) - - # do the successful update twice - host_ip = env.get_virtual_network_host_ip() - update_image( - mender_device, - host_ip, - install_image=valid_image, - scripts=[ensure_persistent_conf] - + storage_device_state_scripts - + scripts_paths, - version=2, - devauth=devauth, - deploy=deploy, - ) - assert mender_device.run("cat %s" % test_log).strip() == "\n".join(scripts) - - # NOTE: With client >= 4.x we only support Artifact version 3 - update_image( - mender_device, - host_ip, - install_image=valid_image, - # Second update should not need storage_device_state_scripts. - scripts=[ensure_persistent_conf] + scripts_paths, - version=3, - devauth=devauth, - deploy=deploy, - ) - assert mender_device.run("cat %s" % test_log).strip() == "\n".join( - scripts - ) + "\n" + "\n".join(scripts) - - finally: - shutil.rmtree(tmpdir) - - -class TestDBMigrationOpenSource(BaseTestDBMigration): - def test_migrate_from_legacy_mender_v1_failure( - self, setup_with_legacy_v1_client, valid_image_with_mender_conf - ): - self.do_test_migrate_from_legacy_mender_v1_failure( - setup_with_legacy_v1_client, valid_image_with_mender_conf - ) - - def test_migrate_from_legacy_mender_v1_success( - self, setup_with_legacy_v1_client, valid_image_with_mender_conf - ): - self.do_test_migrate_from_legacy_mender_v1_success( - setup_with_legacy_v1_client, valid_image_with_mender_conf - ) - - -class TestDBMigrationEnterprise(BaseTestDBMigration): - def test_migrate_from_legacy_mender_v1_failure( - self, enterprise_with_legacy_v1_client, valid_image_with_mender_conf - ): - self.do_test_migrate_from_legacy_mender_v1_failure( - enterprise_with_legacy_v1_client, valid_image_with_mender_conf - ) - - def test_migrate_from_legacy_mender_v1_success( - self, enterprise_with_legacy_v1_client, valid_image_with_mender_conf - ): - self.do_test_migrate_from_legacy_mender_v1_success( - enterprise_with_legacy_v1_client, valid_image_with_mender_conf - ) diff --git a/tests/tests/test_deployment_aborting.py b/tests/tests/test_deployment_aborting.py deleted file mode 100644 index fa80fb983..000000000 --- a/tests/tests/test_deployment_aborting.py +++ /dev/null @@ -1,176 +0,0 @@ -# Copyright 2022 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import pytest - -from ..common_setup import ( - standard_setup_one_client_bootstrapped, - enterprise_one_client_bootstrapped, -) -from .common_update import common_update_procedure -from ..MenderAPI import DeviceAuthV2, Deployments -from .mendertesting import MenderTesting - - -class BaseTestDeploymentAborting(MenderTesting): - def abort_deployment( - self, - container_manager, - install_image, - abort_step=None, - number_of_reboots=0, - ): - """ - Trigger a deployment, and cancel it within 15 seconds, make sure no deployment is performed. - - Args: - number_of_reboots: if set to 0, a manual reboot is performed and - checks are performed. - if set to > 0, wait until device is rebooted - number_of_reboots times. - """ - - mender_device = container_manager.device - - devauth = DeviceAuthV2(container_manager.auth) - deploy = Deployments(container_manager.auth, devauth) - - expected_partition = mender_device.get_active_partition() - expected_image_id = mender_device.yocto_id_installed_on_machine() - host_ip = container_manager.get_virtual_network_host_ip() - with mender_device.get_reboot_detector(host_ip) as reboot: - deployment_id, _ = common_update_procedure( - install_image, - verify_status=False, - devauth=devauth, - deploy=deploy, - ) - - if abort_step is not None: - deploy.check_expected_statistics( - deployment_id, abort_step, 1, polling_frequency=0 - ) - - deploy.abort(deployment_id) - - # there will be abored deployment only if the deployment - # for the device has already started - if abort_step is not None: - deploy.check_expected_statistics(deployment_id, "aborted", 1) - - # no deployment logs are sent by the client, is this expected? - for d in devauth.get_devices(): - deploy.get_logs(d["id"], deployment_id, expected_status=404) - - if number_of_reboots > 0: - # If Mender performs reboot, we need to wait for it to reboot - # back into the original filesystem. - reboot.verify_reboot_performed(number_of_reboots=number_of_reboots) - else: - # Else we reboot ourselves, just to make sure that we have not - # unintentionally switched to the new partition. - reboot.verify_reboot_not_performed() - mender_device.run("( sleep 10 ; reboot ) 2>/dev/null >/dev/null &") - reboot.verify_reboot_performed() - - assert mender_device.get_active_partition() == expected_partition - assert mender_device.yocto_id_installed_on_machine() == expected_image_id - deploy.check_expected_status("finished", deployment_id) - - -class TestDeploymentAbortingOpenSource(BaseTestDeploymentAborting): - @MenderTesting.fast - def test_deployment_abortion_instantly( - self, standard_setup_one_client_bootstrapped, valid_image - ): - self.abort_deployment( - standard_setup_one_client_bootstrapped, valid_image, number_of_reboots=0 - ) - - @MenderTesting.fast - def test_deployment_abortion_downloading( - self, standard_setup_one_client_bootstrapped, valid_image - ): - self.abort_deployment( - standard_setup_one_client_bootstrapped, - valid_image, - "downloading", - number_of_reboots=0, - ) - - @MenderTesting.fast - def test_deployment_abortion_installing( - self, standard_setup_one_client_bootstrapped, valid_image - ): - self.abort_deployment( - standard_setup_one_client_bootstrapped, - valid_image, - "installing", - number_of_reboots=1, - ) - - @MenderTesting.fast - def test_deployment_abortion_rebooting( - self, standard_setup_one_client_bootstrapped, valid_image - ): - self.abort_deployment( - standard_setup_one_client_bootstrapped, - valid_image, - "rebooting", - number_of_reboots=2, - ) - - -class TestDeploymentAbortingEnterprise(BaseTestDeploymentAborting): - @MenderTesting.fast - def test_deployment_abortion_instantly( - self, enterprise_one_client_bootstrapped, valid_image - ): - self.abort_deployment( - enterprise_one_client_bootstrapped, valid_image, number_of_reboots=0 - ) - - @MenderTesting.fast - def test_deployment_abortion_downloading( - self, enterprise_one_client_bootstrapped, valid_image - ): - self.abort_deployment( - enterprise_one_client_bootstrapped, - valid_image, - "downloading", - number_of_reboots=0, - ) - - @MenderTesting.fast - def test_deployment_abortion_installing( - self, enterprise_one_client_bootstrapped, valid_image - ): - self.abort_deployment( - enterprise_one_client_bootstrapped, - valid_image, - "installing", - number_of_reboots=1, - ) - - @MenderTesting.fast - def test_deployment_abortion_rebooting( - self, enterprise_one_client_bootstrapped, valid_image - ): - self.abort_deployment( - enterprise_one_client_bootstrapped, - valid_image, - "rebooting", - number_of_reboots=2, - ) diff --git a/tests/tests/test_deployment_retry.py b/tests/tests/test_deployment_retry.py deleted file mode 100644 index 6ae148e8a..000000000 --- a/tests/tests/test_deployment_retry.py +++ /dev/null @@ -1,134 +0,0 @@ -# Copyright 2022 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import pytest -import tempfile -import uuid - -from .. import conftest -from ..common_setup import enterprise_no_client -from ..MenderAPI import logger, Authentication, DeviceAuthV2, Deployments -from ..helpers import Helpers -from testutils.infra.cli import CliTenantadm -from testutils.common import Tenant, User, new_tenant_client -from .common_artifact import get_script_artifact -from .mendertesting import MenderTesting - - -def make_script_artifact(artifact_name, device_type, output_path): - script = b"""\ -#! /bin/bash - -set -xe - -# Just give it a little bit of time -sleep 6s - -touch /tmp/retry-attempts -attempts=$(cat /tmp/retry-attempts) -attempts=$((attempts+1)) - -# Increment the retry count -echo "$attempts" > /tmp/retry-attempts - -if [[ $attempts -lt 3 ]]; then - exit 1 -fi - -# Successful update after three attempts -exit 0 -""" - return get_script_artifact(script, artifact_name, device_type, output_path) - - -@pytest.mark.usefixtures("enterprise_no_client") -class TestDeploymentRetryEnterprise(MenderTesting): - """Tests the retry deployment functionality""" - - @MenderTesting.fast - def test_deployment_retry_failed_update(self, enterprise_no_client): - """Tests that a client installing a deployment created with a retry limit - - This is done through setting up a new tenant on the enterprise plan, - with a device bootstrapped to the tenant. Then an Artifact is created - which contains a script, for the script update module. The script will - store a retry-count in a temp-file on the device, and fail, as long as - the retry-count < 3. On the third go, the script will, pass, and along - with it, so should the update. - - """ - - env = enterprise_no_client - - # Create an enterprise plan tenant - uuidv4 = str(uuid.uuid4()) - tname = "test.mender.io-{}".format(uuidv4) - email = "some.user+{}@example.com".format(uuidv4) - u = User("", email, "whatsupdoc") - cli = CliTenantadm(containers_namespace=env.name) - tid = cli.create_org(tname, u.name, u.pwd, plan="enterprise") - tenant = cli.get_tenant(tid) - tenant = json.loads(tenant) - ttoken = tenant["tenant_token"] - logger.info(f"tenant json: {tenant}") - tenant = Tenant("tenant", tid, ttoken) - tenant.users.append(u) - - # And authorize the user to the tenant account - auth = Authentication(name="enterprise-tenant", username=u.name, password=u.pwd) - auth.create_org = False - auth.reset_auth_token() - devauth = DeviceAuthV2(auth) - deploy = Deployments(auth, devauth) - - # Add a client to the tenant - device = new_tenant_client( - enterprise_no_client, "mender-client", tenant.tenant_token - ) - devauth.accept_devices(1) - - # Install the script update module required for this test - Helpers.install_community_update_module(device, "script") - - with tempfile.NamedTemporaryFile() as tf: - - artifact = make_script_artifact( - "retry-artifact", conftest.machine_name, tf.name - ) - - deploy.upload_image(artifact) - - devices = list( - set([device["id"] for device in devauth.get_devices_status("accepted")]) - ) - assert len(devices) == 1 - - deployment_id = deploy.trigger_deployment( - "retry-test", artifact_name="retry-artifact", devices=devices, retries=3 - ) - logger.info(deploy.get_deployment(deployment_id)) - - # Now just wait for the update to succeed - deploy.check_expected_statistics(deployment_id, "success", 1) - deploy.check_expected_status("finished", deployment_id) - - # Verify the update was actually installed on the device - out = device.run("mender-update show-artifact").strip() - assert out == "retry-artifact" - - # Verify the number of attempts taken to install the update - out = device.run("cat /tmp/retry-attempts").strip() - assert out == "3" diff --git a/tests/tests/test_docker_compose.py b/tests/tests/test_docker_compose.py deleted file mode 100644 index 788e324ef..000000000 --- a/tests/tests/test_docker_compose.py +++ /dev/null @@ -1,574 +0,0 @@ -# Copyright 2025 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import re -import time -import uuid -import pytest -import subprocess -import tempfile -from contextlib import contextmanager -from dataclasses import dataclass - -from ..common_setup import standard_setup_extended -from .common_update import common_update_procedure -from ..MenderAPI import DeviceAuthV2, Deployments, logger -from .mendertesting import MenderTesting -from testutils.common import requests_get - - -@pytest.fixture(scope="session") -def artifact_gen_script(): - with tempfile.TemporaryDirectory() as temp_dir: - script_path = os.path.join(temp_dir, "gen_docker-compose") - - tag_pattern = re.compile(r"^\d+\.\d+\.\d+(?:-build\d+)?$") - version = os.environ.get("MENDER_CONTAINER_MODULES_VERSION", "main") - - base_url = "https://raw.githubusercontent.com/mendersoftware/mender-container-modules/refs" - file_path = "src/gen_docker-compose" - - if tag_pattern.match(version): - ref_path = f"tags/{version}" - elif version.startswith("pull/"): - ref_path = version - else: - ref_path = f"heads/{version}" - - url = f"{base_url}/{ref_path}/{file_path}" - - req = requests_get(url) - with open(script_path, "w") as f: - f.write(req.text) - - os.chmod(script_path, 0o755) - yield script_path - - -@dataclass -class DockerService: - name: str - image: str - base_image: str = "busybox:latest" - build_image: bool = True - - -@contextmanager -def create_test_manifest(services): - """Create Docker images and compose manifest for testing.""" - with tempfile.TemporaryDirectory() as temp_dir: - manifests_dir = os.path.join(temp_dir, "manifests") - images_dir = os.path.join(temp_dir, "images") - os.makedirs(manifests_dir) - os.makedirs(images_dir) - - # Build and save images for each service that needs it - for service in services: - if service.build_image: - dockerfile = os.path.join(temp_dir, f"Dockerfile.{service.name}") - with open(dockerfile, "w") as f: - # Add a uuid to to each Dockerfile to generate unique images (mainly we can check proper cleanup) - f.write( - f'FROM {service.base_image}\nRUN echo "{uuid.uuid4()}" > /image_id\nCMD ["sleep", "infinity"]\n' - ) - subprocess.check_call( - ["docker", "build", "-t", service.image, "-f", dockerfile, temp_dir] - ) - subprocess.check_call( - [ - "docker", - "save", - "-o", - os.path.join(images_dir, f"{service.image}.tar"), - service.image, - ] - ) - - with open(os.path.join(manifests_dir, "docker-compose.yml"), "w") as f: - f.write("services:\n") - for service in services: - f.write(f" {service.name}:\n") - f.write(f" image: {service.image}\n") - f.write(f" network_mode: bridge\n") - - yield manifests_dir, images_dir - - -def make_docker_compose_artifact( - artifact_gen_script, manifests_dir, project_name, images_dir, extra_args=None -): - def make_artifact(filename, artifact_name): - cmd = [ - artifact_gen_script, - "--artifact-name", - artifact_name, - "--device-type", - "qemux86-64", - "--output-path", - filename, - "--manifests-dir", - manifests_dir, - "--images-dir", - images_dir, - "--project-name", - project_name, - ] - if extra_args: - cmd.extend(extra_args) - subprocess.check_call(cmd) - return filename - - return make_artifact - - -@pytest.mark.min_mender_client_version("6.0.0") -class TestDockerCompose(MenderTesting): - def test_successful_rollback(self, standard_setup_extended, artifact_gen_script): - env = standard_setup_extended - mender_device = env.device - - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - devices = devauth.get_devices_status("accepted") - assert len(devices) == 1 - device_id = devices[0]["id"] - - services = [ - DockerService(name="test1", image="test-container-image1"), - DockerService(name="test2", image="test-container-image2"), - ] - - with create_test_manifest(services) as (manifests_dir, images_dir): - deployment_id, _ = common_update_procedure( - verify_status=True, - devices=[device_id], - make_artifact=make_docker_compose_artifact( - artifact_gen_script, manifests_dir, "test", images_dir - ), - devauth=devauth, - deploy=deploy, - ) - - deploy.check_expected_status("finished", deployment_id) - deploy.check_expected_statistics(deployment_id, "success", 1) - - docker_ps = mender_device.run("docker ps") - logger.info(f"docker ps output after successful deployment:\n{docker_ps}") - assert "test-container-image1" in docker_ps - assert "test-container-image2" in docker_ps - - # Trigger a rollback by providing a non-existing image - services = [ - DockerService(name="test1", image="non-existing-image", build_image=False), - DockerService(name="test2", image="test-container-image3"), - ] - - with create_test_manifest(services) as (manifests_dir, images_dir): - deployment_id, _ = common_update_procedure( - verify_status=True, - devices=[device_id], - make_artifact=make_docker_compose_artifact( - artifact_gen_script, manifests_dir, "test", images_dir - ), - devauth=devauth, - deploy=deploy, - ) - - deploy.check_expected_status("finished", deployment_id) - deploy.check_expected_statistics(deployment_id, "failure", 1) - - docker_ps = mender_device.run("docker ps") - logger.info(f"docker ps output after rollback:\n{docker_ps}") - assert "test-container-image1" in docker_ps - assert "test-container-image2" in docker_ps - assert not "non-existing-image" in docker_ps - assert not "test-container-image3" in docker_ps - - def test_invalid_manifest(self, standard_setup_extended, artifact_gen_script): - env = standard_setup_extended - mender_device = env.device - - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - devices = devauth.get_devices_status("accepted") - assert len(devices) == 1 - device_id = devices[0]["id"] - - with tempfile.TemporaryDirectory() as temp_dir: - manifests_dir = os.path.join(temp_dir, "manifests") - images_dir = os.path.join(temp_dir, "images") - os.makedirs(manifests_dir) - os.makedirs(images_dir) - - with open(os.path.join(manifests_dir, "docker-compose.yml"), "w") as f: - f.write("services:\n") - f.write(" ,,,:\n") # Invalid service name - f.write(" image: test-container-image1\n") - f.write(" network_mode: bridge\n") - - with open(os.path.join(images_dir, "dummy.tar"), "wb") as f: - f.write(b"dummy") - - deployment_id, _ = common_update_procedure( - verify_status=True, - devices=[device_id], - make_artifact=make_docker_compose_artifact( - artifact_gen_script, manifests_dir, "test", images_dir - ), - devauth=devauth, - deploy=deploy, - ) - - deploy.check_expected_status("finished", deployment_id) - deploy.check_expected_statistics(deployment_id, "failure", 1) - - def test_empty_image_tarball(self, standard_setup_extended, artifact_gen_script): - env = standard_setup_extended - mender_device = env.device - - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - devices = devauth.get_devices_status("accepted") - assert len(devices) == 1 - device_id = devices[0]["id"] - - services = [ - DockerService(name="test1", image="test-image1"), - ] - - with create_test_manifest(services) as (manifests_dir, images_dir): - # Corrupt the tarball by overwriting with empty file - corrupted_tar = os.path.join(images_dir, "test-image1.tar") - with open(corrupted_tar, "wb") as f: - f.write(b"") - - deployment_id, _ = common_update_procedure( - verify_status=True, - devices=[device_id], - make_artifact=make_docker_compose_artifact( - artifact_gen_script, manifests_dir, "test", images_dir - ), - devauth=devauth, - deploy=deploy, - ) - - deploy.check_expected_status("finished", deployment_id) - deploy.check_expected_statistics(deployment_id, "failure", 1) - - def test_consecutive_updates(self, standard_setup_extended, artifact_gen_script): - env = standard_setup_extended - mender_device = env.device - - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - devices = devauth.get_devices_status("accepted") - assert len(devices) == 1 - device_id = devices[0]["id"] - - # First deployment - services = [ - DockerService(name="test1", image="test-container-image1"), - DockerService(name="test2", image="test-container-image2"), - ] - - with create_test_manifest(services) as (manifests_dir, images_dir): - deployment_id, _ = common_update_procedure( - verify_status=True, - devices=[device_id], - make_artifact=make_docker_compose_artifact( - artifact_gen_script, manifests_dir, "test", images_dir - ), - devauth=devauth, - deploy=deploy, - ) - - deploy.check_expected_status("finished", deployment_id) - deploy.check_expected_statistics(deployment_id, "success", 1) - - docker_ps = mender_device.run("docker ps") - logger.info(f"docker ps output after first deployment:\n{docker_ps}") - assert "test-container-image1" in docker_ps - assert "test-container-image2" in docker_ps - - # Verify images are loaded - docker_images = mender_device.run("docker image ls") - assert "test-container-image1" in docker_images - assert "test-container-image2" in docker_images - - # Second deployment with different services - services = [ - DockerService(name="test3", image="test-container-image3"), - DockerService(name="test4", image="test-container-image4"), - ] - - with create_test_manifest(services) as (manifests_dir, images_dir): - deployment_id, _ = common_update_procedure( - verify_status=True, - devices=[device_id], - make_artifact=make_docker_compose_artifact( - artifact_gen_script, manifests_dir, "test", images_dir - ), - devauth=devauth, - deploy=deploy, - ) - - deploy.check_expected_status("finished", deployment_id) - deploy.check_expected_statistics(deployment_id, "success", 1) - - docker_ps = mender_device.run("docker ps") - logger.info(f"docker ps output after second deployment:\n{docker_ps}") - # Old containers should be cleaned up - assert "test-container-image1" not in docker_ps - assert "test-container-image2" not in docker_ps - # New containers should be running - assert "test-container-image3" in docker_ps - assert "test-container-image4" in docker_ps - - # Verify old images are cleaned up - docker_images = mender_device.run("docker images") - assert "test-container-image1" not in docker_images - assert "test-container-image2" not in docker_images - assert "test-container-image3" in docker_images - assert "test-container-image4" in docker_images - - def test_rollback_with_broken_connection( - self, standard_setup_extended, artifact_gen_script - ): - """Test rollback when server connection is broken during deployment.""" - env = standard_setup_extended - mender_device = env.device - - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - devices = devauth.get_devices_status("accepted") - assert len(devices) == 1 - device_id = devices[0]["id"] - - services = [ - DockerService(name="test1", image="test-container-image1"), - ] - - with create_test_manifest(services) as (manifests_dir, images_dir): - deployment_id, _ = common_update_procedure( - verify_status=True, - devices=[device_id], - make_artifact=make_docker_compose_artifact( - artifact_gen_script, manifests_dir, "test", images_dir - ), - devauth=devauth, - deploy=deploy, - ) - - deploy.check_expected_status("finished", deployment_id) - deploy.check_expected_statistics(deployment_id, "success", 1) - - docker_ps = mender_device.run("docker ps") - logger.info(f"docker ps output after initial deployment:\n{docker_ps}") - assert "test-container-image1" in docker_ps - - services = [ - DockerService(name="test2", image="test-container-image2"), - ] - - with create_test_manifest(services) as (manifests_dir, images_dir): - # Create an Artifact state script that blocks network access to the Mender server - with tempfile.TemporaryDirectory() as script_dir: - script_path = os.path.join(script_dir, "ArtifactInstall_Leave_00") - with open(script_path, "w") as script_file: - script_file.write("""#!/bin/sh -# Block network access to simulate connection loss by redirecting to invalid IP -echo "Blocking network connection to Mender server" -sed -i.backup -e '$a127.0.0.1 docker.mender.io' /etc/hosts -exit 0 -""") - os.chmod(script_path, 0o755) - - deployment_id, _ = common_update_procedure( - verify_status=True, - devices=[device_id], - make_artifact=make_docker_compose_artifact( - artifact_gen_script, - manifests_dir, - "test", - images_dir, - extra_args=["--", "--script", script_path], - ), - devauth=devauth, - deploy=deploy, - ) - logger.info("Waiting for retry status update to fail (takes ~3 minutes)") - - timeout = 240 - for _ in range(timeout): - if "Giving up on sending status updates to server" in mender_device.run( - "systemctl status mender-updated", hide=True - ): - break - logger.debug("Waiting for status update to give up") - time.sleep(1) - mender_device.run("mv /etc/hosts.backup /etc/hosts") - - deploy.check_expected_status("finished", deployment_id) - deploy.check_expected_statistics(deployment_id, "failure", 1) - - # Verify rollback occurred - original service should still be running - docker_ps = mender_device.run("docker ps") - logger.info(f"docker ps output after failed deployment:\n{docker_ps}") - assert "test-container-image1" in docker_ps - assert "test-container-image2" not in docker_ps - - def test_healthcheck(self, standard_setup_extended, artifact_gen_script): - env = standard_setup_extended - mender_device = env.device - - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - devices = devauth.get_devices_status("accepted") - assert len(devices) == 1 - device_id = devices[0]["id"] - - with tempfile.TemporaryDirectory() as temp_dir: - manifests_dir = os.path.join(temp_dir, "manifests") - images_dir = os.path.join(temp_dir, "images") - os.makedirs(manifests_dir) - os.makedirs(images_dir) - - # Build and save the image - dockerfile = os.path.join(temp_dir, "Dockerfile.test1") - with open(dockerfile, "w") as f: - f.write( - f'FROM busybox:latest\nRUN echo "{uuid.uuid4()}" > /image_id\nCMD ["sleep", "infinity"]\n' - ) - subprocess.check_call( - [ - "docker", - "build", - "-t", - "test-container-image1", - "-f", - dockerfile, - temp_dir, - ] - ) - subprocess.check_call( - [ - "docker", - "save", - "-o", - os.path.join(images_dir, "test-container-image1.tar"), - "test-container-image1", - ] - ) - - # Create manifest with passing healthcheck - with open(os.path.join(manifests_dir, "docker-compose.yml"), "w") as f: - f.write("services:\n") - f.write(" test1:\n") - f.write(" image: test-container-image1\n") - f.write(" network_mode: bridge\n") - f.write(" healthcheck:\n") - f.write(' test: ["CMD", "true"]\n') - f.write(" interval: 1s\n") - f.write(" timeout: 1s\n") - f.write(" retries: 1\n") - - deployment_id, _ = common_update_procedure( - verify_status=True, - devices=[device_id], - make_artifact=make_docker_compose_artifact( - artifact_gen_script, manifests_dir, "test", images_dir - ), - devauth=devauth, - deploy=deploy, - ) - - deploy.check_expected_status("finished", deployment_id) - deploy.check_expected_statistics(deployment_id, "success", 1) - - docker_ps = mender_device.run("docker ps") - logger.info(f"docker ps output after successful deployment:\n{docker_ps}") - assert "test-container-image1" in docker_ps - - with tempfile.TemporaryDirectory() as temp_dir: - manifests_dir = os.path.join(temp_dir, "manifests") - images_dir = os.path.join(temp_dir, "images") - os.makedirs(manifests_dir) - os.makedirs(images_dir) - - # Build and save the image - dockerfile = os.path.join(temp_dir, "Dockerfile.test2") - with open(dockerfile, "w") as f: - f.write( - f'FROM busybox:latest\nRUN echo "{uuid.uuid4()}" > /image_id\nCMD ["sleep", "infinity"]\n' - ) - subprocess.check_call( - [ - "docker", - "build", - "-t", - "test-container-image2", - "-f", - dockerfile, - temp_dir, - ] - ) - subprocess.check_call( - [ - "docker", - "save", - "-o", - os.path.join(images_dir, "test-container-image2.tar"), - "test-container-image2", - ] - ) - - # Create manifest with failing healthcheck - with open(os.path.join(manifests_dir, "docker-compose.yml"), "w") as f: - f.write("services:\n") - f.write(" test2:\n") - f.write(" image: test-container-image2\n") - f.write(" network_mode: bridge\n") - f.write(" healthcheck:\n") - f.write(' test: ["CMD", "false"]\n') - f.write(" interval: 1s\n") - f.write(" timeout: 1s\n") - f.write(" retries: 1\n") - - deployment_id, _ = common_update_procedure( - verify_status=True, - devices=[device_id], - make_artifact=make_docker_compose_artifact( - artifact_gen_script, manifests_dir, "test", images_dir - ), - devauth=devauth, - deploy=deploy, - ) - - deploy.check_expected_status("finished", deployment_id) - deploy.check_expected_statistics(deployment_id, "failure", 1) - - # Verify rollback occurred - original service should still be running - docker_ps = mender_device.run("docker ps") - logger.info( - f"docker ps output after failed healthcheck deployment:\n{docker_ps}" - ) - assert "test-container-image1" in docker_ps - assert "test-container-image2" not in docker_ps diff --git a/tests/tests/test_fault_tolerance.py b/tests/tests/test_fault_tolerance.py deleted file mode 100644 index ecb5cd107..000000000 --- a/tests/tests/test_fault_tolerance.py +++ /dev/null @@ -1,628 +0,0 @@ -# Copyright 2023 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import pytest -import shutil -import tempfile -import time - -from .. import conftest -from ..common_setup import ( - standard_setup_one_client_bootstrapped, - enterprise_one_client_bootstrapped, -) -from .common_update import common_update_procedure, update_image_failed -from ..MenderAPI import DeviceAuthV2, Deployments, logger -from .mendertesting import MenderTesting - - -class BasicTestFaultTolerance(MenderTesting): - def manipulate_network_connectivity( - self, - device, - accessible, - hosts=["mender-artifact-storage.localhost", "mender-api-gateway"], - ): - try: - for h in hosts: - if h == "s3.docker.mender.io": - self.block_by_domain(device, accessible, h) - else: - self.block_by_ip(device, accessible, h) - except Exception as e: - logger.info("Exception while messing with network connectivity: %s", str(e)) - - def block_by_ip(self, device, accessible, host): - """Get IP of host and block by that.""" - gateway_ip = device.run( - r"nslookup %s | grep -A1 'Name:' | grep -E '^Address( 1)?:' | grep -oE '((1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])'" - % (host), - hide=True, - ).strip() - - if accessible: - logger.info("Allowing network communication to %s" % host) - device.run("iptables -D INPUT -s %s -j DROP" % (gateway_ip), hide=True) - device.run("iptables -D OUTPUT -s %s -j DROP" % (gateway_ip), hide=True) - else: - logger.info("Disallowing network communication to %s" % host) - device.run("iptables -I INPUT 1 -s %s -j DROP" % gateway_ip, hide=True) - device.run("iptables -I OUTPUT 1 -s %s -j DROP" % gateway_ip, hide=True) - - def block_by_domain(self, device, accessible, host): - """Some services shouldn't be blocked by ip, because they share it with other services. - (example: s3 is the same IP as gateway as a whole). - Block these by host/domain name instead (using iptables string matching). - """ - - # extra modules for iptables string matching - device.run("modprobe xt_string") - device.run("modprobe ts-bm") - - if accessible: - logger.info("Allowing network communication to %s" % host) - device.run( - "iptables -D INPUT -p tcp -m string --algo bm --string %s -j REJECT --reject-with tcp-reset" - % host - ) - - device.run( - "iptables -D OUTPUT -p tcp -m string --algo bm --string %s -j REJECT --reject-with tcp-reset" - % host - ) - else: - logger.info("Disallowing network communication to %s" % host) - device.run( - "iptables -I INPUT -p tcp -m string --algo bm --string %s -j REJECT --reject-with tcp-reset" - % host, - hide=True, - ) - device.run( - "iptables -I OUTPUT -p tcp -m string --algo bm --string %s -j REJECT --reject-with tcp-reset" - % host, - hide=True, - ) - - def wait_for_download_retry_attempts( - self, device, search_string, num_retries=2, timeout=10 - ): - """Block until logs contain messages related to failed download attempts""" - - timeout_time = int(time.time()) + ( - timeout * 90 - ) # 90s per each 60s retry to give time for a connection timeout to occur each time - start_time = int(time.time()) - num_retries_attempted = 0 - - while int(time.time()) < timeout_time: - output = device.run( - f'journalctl --unit mender-updated --full --no-pager | grep -E \'name="http_resumer:client" msg=".*{search_string}\' | wc -l', - hide=True, - ) - time.sleep(2) - if int(output) >= num_retries: # check that some retries have occurred - logger.info( - f"Looks like the download was retried {num_retries} times, restoring download functionality" - ) - num_retries_attempted = int(output) - break - num_retries_attempted = int(output) - - # if num_retries expected is smaller then timeout, we expect success. - # if it's bigger, then we expect a timeout - if num_retries < timeout: - if timeout_time <= int(time.time()): - pytest.fail("timed out waiting for download retries") - # make sure that retries happen after 'num_retries' minutes have passed - assert ( - int(time.time()) - start_time - > (num_retries - 1) - * 60 # need to decrease by 1 because the first retry happens almost immediately, not after a minute - ), f"Ooops, looks like the retry happened within less than {num_retries} minutes" - - logger.info("Waiting for system to finish download") - return num_retries_attempted - - def do_test_update_image_breaks_networking( - self, - env, - broken_network_image, - ): - """ - Install an image without systemd-networkd binary existing. - The network will not function, mender will not be able to send any logs. - - The expected status is the update will rollback, and be considered a failure - """ - - mender_device = env.device - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - host_ip = env.get_virtual_network_host_ip() - with mender_device.get_reboot_detector(host_ip) as reboot: - deployment_id, _ = common_update_procedure( - broken_network_image, devauth=devauth, deploy=deploy - ) - reboot.verify_reboot_performed() # since the network is broken, two reboots will be performed, and the last one will be detected - deploy.check_expected_statistics(deployment_id, "failure", 1) - - def do_test_deployed_during_network_outage( - self, - env, - valid_image_with_mender_conf, - ): - """ - Install a valid upgrade image while there is no network availability on the device - Re-establishing the network connectivity results in the upgrade to be triggered. - - Emulate a flaky network connection, and ensure that the deployment still succeeds. - """ - - mender_device = env.device - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - self.manipulate_network_connectivity(mender_device, False) - - host_ip = env.get_virtual_network_host_ip() - with mender_device.get_reboot_detector(host_ip) as reboot: - mender_conf = mender_device.run("cat /etc/mender/mender.conf") - deployment_id, expected_yocto_id = common_update_procedure( - valid_image_with_mender_conf(mender_conf), - verify_status=False, - devauth=devauth, - deploy=deploy, - ) - time.sleep(60) - - for i in range(5): - time.sleep(5) - self.manipulate_network_connectivity(mender_device, i % 2 == 0) - self.manipulate_network_connectivity(mender_device, True) - - logger.info("Network stabilized") - reboot.verify_reboot_performed() - deploy.check_expected_statistics(deployment_id, "success", 1) - - assert mender_device.yocto_id_installed_on_machine() == expected_yocto_id - - def do_test_image_download_retry_timeout( - self, - env, - valid_image_with_mender_conf, - ): - """ - Install an update, and block storage connection when we detect it's - being copied over to the inactive partition. - - The test should result in a successful download retry. - - NOTE: storage and gateway share an ip, so disabling connectivity - is tricky - we must alternate between blocking by the whole ip and blocking - just by domain - """ - - mender_device = env.device - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - # make tcp timeout quicker, none persistent changes - mender_device.run("echo 2 > /proc/sys/net/ipv4/tcp_keepalive_time") - mender_device.run("echo 2 > /proc/sys/net/ipv4/tcp_keepalive_intvl") - mender_device.run("echo 3 > /proc/sys/net/ipv4/tcp_syn_retries") - - # to speed up timeouting client connection - mender_device.run("echo 1 > /proc/sys/net/ipv4/tcp_keepalive_probes") - - inactive_part = mender_device.get_passive_partition() - - host_ip = env.get_virtual_network_host_ip() - - blocked_service = None - - with mender_device.get_reboot_detector(host_ip) as reboot: - # Block after we start the download. - mender_conf = mender_device.run("cat /etc/mender/mender.conf") - deployment_id, new_yocto_id = common_update_procedure( - valid_image_with_mender_conf(mender_conf), - devauth=devauth, - deploy=deploy, - ) - mender_device.run( - "start=$(date -u +%s);" - + 'output="";' - + 'while [ -z "$output" ]; do ' - + "sleep 0.1;" - + 'output="$(fuser -mv %s)";' % inactive_part - + "now=$(date -u +%s);" - + "if [ $(($now - $start)) -gt 600 ]; then " - + "exit 1;" - + "fi;" - + "done", - wait=10 * 60, - ) - - # storage must be blocked by ip to kill an ongoing connection - # so block the whole gateway - blocked_service = "docker.mender.io" - - self.manipulate_network_connectivity( - mender_device, False, hosts=[blocked_service] - ) - - # re-enable connectivity after 2 retries - self.wait_for_download_retry_attempts(mender_device, "Connection timed out") - - self.manipulate_network_connectivity( - mender_device, True, hosts=[blocked_service] - ) - - reboot.verify_reboot_performed() - deploy.check_expected_status("finished", deployment_id) - - assert mender_device.get_active_partition() == inactive_part - assert mender_device.yocto_id_installed_on_machine() == new_yocto_id - reboot.verify_reboot_not_performed() - - def do_test_image_download_retry_download_count( - self, - env, - valid_image_with_mender_conf, - max_retries, - unsuccessful_retries, - success, - ): - """ - Install an update, and block storage connection when we detect it's - being copied over to the inactive partition - parametrized number of times equal to "unsuccessful_retries" - - The test should result in a successful download retry if "max_retries" >= "unsuccessful_retries". - - NOTE: storage and gateway share an ip, so disabling connectivity - is tricky - we must alternate between blocking by the whole ip and blocking - just by domain - """ - - mender_device = env.device - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - # modify device config with our number of retries - try: - tmpdir = tempfile.mkdtemp() - # retrieve the original configuration file - output = mender_device.run("cat /etc/mender/mender.conf") - config = json.loads(output) - # add RetryDownloadCount value, modifying the default "10" - config["RetryDownloadCount"] = max_retries - mender_conf = os.path.join(tmpdir, "mender.conf") - with open(mender_conf, "w") as fd: - json.dump(config, fd) - env.device.put( - os.path.basename(mender_conf), - local_path=os.path.dirname(mender_conf), - remote_path="/etc/mender", - ) - finally: - shutil.rmtree(tmpdir) - - # start the Mender client - logger.info("Restarting the client with updated configuration.") - env.device.run("systemctl restart mender-updated") - # end of client config modification - - # make tcp timeout quicker, none persistent changes - mender_device.run("echo 2 > /proc/sys/net/ipv4/tcp_keepalive_time") - mender_device.run("echo 2 > /proc/sys/net/ipv4/tcp_keepalive_intvl") - mender_device.run("echo 3 > /proc/sys/net/ipv4/tcp_syn_retries") - - # to speed up timeouting client connection - mender_device.run("echo 1 > /proc/sys/net/ipv4/tcp_keepalive_probes") - - active_part = mender_device.get_active_partition() - inactive_part = mender_device.get_passive_partition() - old_yocto_id = mender_device.yocto_id_installed_on_machine() - - host_ip = env.get_virtual_network_host_ip() - - blocked_service = None - - with mender_device.get_reboot_detector(host_ip) as reboot: - # Block after we start the download. - mender_conf = mender_device.run("cat /etc/mender/mender.conf") - deployment_id, new_yocto_id = common_update_procedure( - valid_image_with_mender_conf(mender_conf), - devauth=devauth, - deploy=deploy, - ) - mender_device.run( - "start=$(date -u +%s);" - + 'output="";' - + 'while [ -z "$output" ]; do ' - + "sleep 0.1;" - + 'output="$(fuser -mv %s)";' % inactive_part - + "now=$(date -u +%s);" - + "if [ $(($now - $start)) -gt 600 ]; then " - + "exit 1;" - + "fi;" - + "done", - wait=10 * 60, - ) - - # storage must be blocked by ip to kill an ongoing connection - # so block the whole gateway - blocked_service = "docker.mender.io" - - self.manipulate_network_connectivity( - mender_device, False, hosts=[blocked_service] - ) - - # re-enable connectivity after "unsuccessful_retries" retries - # wait for >= unusccessful_retries retries. If max_retries is smaller, it will timeout as expected - retries_attempted = self.wait_for_download_retry_attempts( - mender_device, - "Resuming download after", - unsuccessful_retries, - max_retries, - ) - if max_retries > unsuccessful_retries: - assert retries_attempted == unsuccessful_retries - else: - assert retries_attempted == max_retries - - self.manipulate_network_connectivity( - mender_device, True, hosts=[blocked_service] - ) - - if success: - reboot.verify_reboot_performed() - deploy.check_expected_status("finished", deployment_id) - - assert mender_device.get_active_partition() == inactive_part - assert mender_device.yocto_id_installed_on_machine() == new_yocto_id - reboot.verify_reboot_not_performed() - else: - reboot.verify_reboot_not_performed() - deploy.check_expected_status("inprogress", deployment_id) - - assert mender_device.get_active_partition() == active_part - assert mender_device.yocto_id_installed_on_machine() == old_yocto_id - - def do_test_image_download_retry_hosts_broken( - self, - env, - valid_image_with_mender_conf, - ): - """ - Block storage host (minio) by modifying the hosts file. - """ - - mender_device = env.device - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - inactive_part = mender_device.get_passive_partition() - - mender_device.run( - "echo '1.1.1.1 s3.docker.mender.io' >> /etc/hosts" - ) # break s3 connectivity before triggering deployment - - host_ip = env.get_virtual_network_host_ip() - with mender_device.get_reboot_detector(host_ip) as reboot: - mender_conf = mender_device.run("cat /etc/mender/mender.conf") - deployment_id, new_yocto_id = common_update_procedure( - valid_image_with_mender_conf(mender_conf), - devauth=devauth, - deploy=deploy, - ) - - self.wait_for_download_retry_attempts( - mender_device, - "Resuming download after ", - ) - mender_device.run("sed -i.bak '/1.1.1.1/d' /etc/hosts") - - reboot.verify_reboot_performed() - deploy.check_expected_status("finished", deployment_id) - - assert mender_device.get_active_partition() == inactive_part - assert mender_device.yocto_id_installed_on_machine() == new_yocto_id - reboot.verify_reboot_not_performed() - - def do_test_rootfs_conf_missing_from_new_update( - self, env, valid_image_with_mender_conf - ): - """Test that the client is able to reboot to roll back if module or rootfs - config is missing from the new partition. This only works for cases where a - reboot restores the state.""" - - mender_device = env.device - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - output = mender_device.run( - "test -e /data/mender/mender.conf && echo true", hide=True - ) - if output.rstrip() != "true": - pytest.skip("Needs split mender.conf configuration to run this test") - - tmpdir = tempfile.mkdtemp() - try: - # With the persistent mender.conf in /data, and the transient - # mender.conf in /etc, we can simply delete the former (rootfs - # config) to break the config, and add it back into the transient - # one to keep the config valid for the existing artifact (but not - # the new one). - output = mender_device.run("cat /data/mender/mender.conf") - persistent_conf = json.loads(output) - mender_device.run("rm /data/mender/mender.conf") - - output = mender_device.run("cat /etc/mender/mender.conf") - conf = json.loads(output) - - conf["RootfsPartA"] = persistent_conf["RootfsPartA"] - conf["RootfsPartB"] = persistent_conf["RootfsPartB"] - - mender_conf = os.path.join(tmpdir, "mender.conf") - with open(mender_conf, "w") as fd: - json.dump(conf, fd) - mender_device.put( - os.path.basename(mender_conf), - local_path=os.path.dirname(mender_conf), - remote_path="/etc/mender", - ) - - host_ip = env.get_virtual_network_host_ip() - update_image_failed( - mender_device, - host_ip, - expected_log_message="Cannot parse RootfsPartA/B in any configuration file!", - install_image=valid_image_with_mender_conf(output), - devauth=devauth, - deploy=deploy, - ) - - finally: - shutil.rmtree(tmpdir) - - -class TestFaultToleranceOpenSource(BasicTestFaultTolerance): - @MenderTesting.slow - def test_update_image_breaks_networking( - self, - standard_setup_one_client_bootstrapped, - broken_network_image, - ): - self.do_test_update_image_breaks_networking( - standard_setup_one_client_bootstrapped, - broken_network_image, - ) - - @MenderTesting.slow - def test_deployed_during_network_outage( - self, - standard_setup_one_client_bootstrapped, - valid_image_with_mender_conf, - ): - self.do_test_deployed_during_network_outage( - standard_setup_one_client_bootstrapped, valid_image_with_mender_conf - ) - - @MenderTesting.slow - def test_image_download_retry_timeout( - self, - standard_setup_one_client_bootstrapped, - valid_image_with_mender_conf, - ): - self.do_test_image_download_retry_timeout( - standard_setup_one_client_bootstrapped, - valid_image_with_mender_conf, - ) - - @pytest.mark.parametrize( - "max_retries, unsuccessful_retries, success", - [(5, 2, True), (5, 7, False), (15, 12, True), (11, 15, False)], - ids=[ - "reducedRetriesSuccess", - "reducedRetriesFailure", - "increasedRetriesSuccess", - "increasedRetriesFailure", - ], - ) - @MenderTesting.slow - def test_image_download_retry_download_count( - self, - standard_setup_one_client_bootstrapped, - valid_image_with_mender_conf, - max_retries, - unsuccessful_retries, - success, - ): - self.do_test_image_download_retry_download_count( - standard_setup_one_client_bootstrapped, - valid_image_with_mender_conf, - max_retries, - unsuccessful_retries, - success, - ) - - @MenderTesting.slow - def test_image_download_retry_hosts_broken( - self, - standard_setup_one_client_bootstrapped, - valid_image_with_mender_conf, - ): - self.do_test_image_download_retry_hosts_broken( - standard_setup_one_client_bootstrapped, valid_image_with_mender_conf - ) - - def test_rootfs_conf_missing_from_new_update( - self, standard_setup_one_client_bootstrapped, valid_image_with_mender_conf - ): - self.do_test_rootfs_conf_missing_from_new_update( - standard_setup_one_client_bootstrapped, valid_image_with_mender_conf - ) - - -class TestFaultToleranceEnterprise(BasicTestFaultTolerance): - @MenderTesting.slow - def test_update_image_breaks_networking( - self, - enterprise_one_client_bootstrapped, - broken_network_image, - ): - self.do_test_update_image_breaks_networking( - enterprise_one_client_bootstrapped, - broken_network_image, - ) - - @MenderTesting.slow - def test_deployed_during_network_outage( - self, - enterprise_one_client_bootstrapped, - valid_image_with_mender_conf, - ): - self.do_test_deployed_during_network_outage( - enterprise_one_client_bootstrapped, valid_image_with_mender_conf - ) - - @MenderTesting.slow - def test_image_download_retry_timeout( - self, - enterprise_one_client_bootstrapped, - valid_image_with_mender_conf, - ): - self.do_test_image_download_retry_timeout( - enterprise_one_client_bootstrapped, valid_image_with_mender_conf - ) - - @MenderTesting.slow - def test_image_download_retry_hosts_broken( - self, - enterprise_one_client_bootstrapped, - valid_image_with_mender_conf, - ): - self.do_test_image_download_retry_hosts_broken( - enterprise_one_client_bootstrapped, valid_image_with_mender_conf - ) - - def test_rootfs_conf_missing_from_new_update( - self, enterprise_one_client_bootstrapped, valid_image_with_mender_conf - ): - self.do_test_rootfs_conf_missing_from_new_update( - enterprise_one_client_bootstrapped, valid_image_with_mender_conf - ) diff --git a/tests/tests/test_filetransfer.py b/tests/tests/test_filetransfer.py deleted file mode 100644 index 17221fc4b..000000000 --- a/tests/tests/test_filetransfer.py +++ /dev/null @@ -1,781 +0,0 @@ -# Copyright 2023 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import json -import io -import os -import os.path - -import requests -import shutil -import tempfile -import pytest -import time -import urllib.parse -import random - -from flaky import flaky - -from tempfile import NamedTemporaryFile - -from ..common_setup import ( - enterprise_one_docker_client_bootstrapped, - standard_setup_one_docker_client_bootstrapped, -) - -from ..MenderAPI import ( - auth, - authentication, - get_container_manager, - reset_mender_api, - logger, - devauth, -) -from .common_connect import prepare_env_for_connect, wait_for_connect -from .common import md5sum -from .mendertesting import MenderTesting -from testutils.infra.container_manager import factory -from testutils.infra.device import MenderDevice - -container_factory = factory.get_factory() -connect_service_name = "mender-connect" - - -def random_filename(): - return "".join(random.choices([chr(c) for c in range(97, 123)], k=8)) - - -def download_file(path, devid, authtoken): - deviceconnect_url = ( - "https://%s/api/management/v1/deviceconnect" - % get_container_manager().get_mender_gateway() - ) - download_url = "%s/devices/%s/download" % (deviceconnect_url, devid) - download_url_with_path = download_url + "?path=" + urllib.parse.quote(path) - return requests.get(download_url_with_path, verify=False, headers=authtoken) - - -def upload_file(path, file, devid, authtoken, mode="600", uid="0", gid="0"): - files = ( - ("path", (None, path)), - ("mode", (None, mode)), - ("uid", (None, uid)), - ("gid", (None, gid)), - ( - "file", - (os.path.basename(path), file, "application/octet-stream"), - ), - ) - deviceconnect_url = ( - "https://%s/api/management/v1/deviceconnect" - % get_container_manager().get_mender_gateway() - ) - upload_url = "%s/devices/%s/upload" % (deviceconnect_url, devid) - return requests.put(upload_url, verify=False, headers=authtoken, files=files) - - -def set_limits(docker_env, mender_device, limits, auth, devid): - tmpdir = tempfile.mkdtemp() - try: - # retrieve the original configuration file - output = mender_device.run("cat /etc/mender/mender-connect.conf") - config = json.loads(output) - # update mender-connect.conf setting the file transfer limits - config["Limits"] = limits - mender_connect_conf = os.path.join(tmpdir, "mender-connect.conf") - with open(mender_connect_conf, "w") as fd: - json.dump(config, fd) - mender_device.run( - "cp /etc/mender/mender-connect.conf /etc/mender/mender-connect.conf-backup-`ls /etc/mender/mender-connect.* | wc -l`" - ) - mender_device.put( - os.path.basename(mender_connect_conf), - local_path=os.path.dirname(mender_connect_conf), - remote_path="/etc/mender", - ) - finally: - shutil.rmtree(tmpdir) - mender_device.run("kill -TERM `pidof %s`" % connect_service_name) - time.sleep(4) - wait_for_connect(auth, devid) - debugoutput = mender_device.run("cat /etc/mender/mender-connect.conf") - logger.info("/etc/mender/mender-connect.conf:\n%s" % debugoutput) - debugoutput = mender_device.run("ls -al /etc/mender") - logger.info("ls -al /etc/mender/:\n%s" % debugoutput) - - -class BaseTestFileTransferDownload(MenderTesting): - def test_download_ok(self, mender_device_setup, content_assertion=None): - # download a file and check its content - path = "/etc/mender/mender.conf" - r = download_file(path, self.devid, self.auth_token) - - assert r.status_code == 200, r.json() - if content_assertion: - assert content_assertion in str(r.content) - assert ( - r.headers.get("Content-Disposition") - == 'attachment; filename="' + os.path.basename(path) + '"' - ) - assert r.headers.get("Content-Type") == "application/octet-stream" - assert r.headers.get("X-Men-File-Gid") == "0" - assert r.headers.get("X-Men-File-Uid") == "0" - assert ( - r.headers.get("X-Men-File-Mode") == "644" - ) # this used to be 600 but in the qemu, it is 644 in mender docker client see QA-1527 - assert r.headers.get("X-Men-File-Path") == "/etc/mender/mender.conf" - assert r.headers.get("X-Men-File-Size") != "" - - def test_download_error_relative_path(self, mender_device_setup): - - # wrong request, path is relative - r = download_file("relative/path", self.devid, self.auth_token) - assert r.status_code == 400, r.json() - assert "path: must be absolute" in r.json().get("error") - - def test_upload_error_no_such_file_or_directory(self, mender_device_setup): - - # wrong request, no such file or directory - path = "/does/not/exist" - r = download_file(path, self.devid, self.auth_token) - assert r.status_code == 400, r.json() - assert "/does/not/exist: no such file or directory" in r.json().get("error") - - def test_upload_and_download_ok(self, mender_device_setup): - - try: - # create a 40MB random file - f = NamedTemporaryFile(delete=False) - for i in range(4 * 1024): - f.write(os.urandom(1024)) - f.close() - - uid = random.randint(100, 200) - gid = random.randint(100, 200) - - fname = random_filename() - path = f"/tmp/{fname}" - r = upload_file( - path, - open(f.name, "rb"), - self.devid, - self.auth_token, - mode="600", - uid=str(uid), - gid=str(gid), - ) - assert r.status_code == 201, r.json() - - # download the file - r = download_file(path, self.devid, self.auth_token) - assert r.status_code == 200, r.json() - assert ( - r.headers.get("Content-Disposition") - == f'attachment; filename="{fname}"' - ) - assert r.headers.get("Content-Type") == "application/octet-stream" - assert r.headers.get("X-Men-File-Mode") == "600" - assert r.headers.get("X-Men-File-Uid") == str(uid) - assert r.headers.get("X-Men-File-Gid") == str(gid) - assert r.headers.get("X-Men-File-Path") == f"/tmp/{fname}" - assert r.headers.get("X-Men-File-Size") == str(4 * 1024 * 1024) - - filename_download = f.name + ".download" - with open(filename_download, "wb") as fw: - fw.write(r.content) - - # verify the file is not corrupted - assert md5sum(filename_download) == md5sum(f.name) - finally: - os.unlink(f.name) - if os.path.isfile(f.name + ".download"): - os.unlink(f.name + ".download") - - def test_upload_error_path_is_relative( - self, - mender_device_setup, - ): - - # wrong request, path is relative - r = upload_file( - "relative/path/dummy.txt", - io.StringIO("dummy"), - self.devid, - self.auth_token, - mode="600", - uid="0", - gid="0", - ) - assert r.status_code == 400, r.json() - assert "path: must be absolute" in r.json().get("error") - - def test_upload_error_file_does_not_exist(self, mender_device_setup): - - r = upload_file( - "/does/not/exist/dummy.txt", - io.StringIO("dummy"), - self.devid, - self.auth_token, - mode="600", - uid="0", - gid="0", - ) - assert r.status_code == 400, r.json() - assert "failed to create target file" in r.json().get("error") - - -class BaseTestFileTransferLimits(MenderTesting): - def test_upload_limits_err_file_outside_chroot(self, mender_device_setup): - "File Transfer limits: file outside chroot; upload forbidden" - - set_limits( - self.env, - self.mender_device, - { - "Enabled": True, - "FileTransfer": {"Chroot": "/var/lib/mender/filetransfer"}, - }, - self.auth, - self.devid, - ) - self.mender_device.run("mkdir -p /var/lib/mender/filetransfer") - - f = NamedTemporaryFile(delete=False) - for i in range(4 * 1024): - f.write(os.urandom(1024)) - f.close() - - fname = random_filename() - r = upload_file( - f"/usr/{fname}", - open(f.name, "rb"), - self.devid, - self.auth_token, - ) - - assert r.status_code == 400, r.json() - assert ( - r.json().get("error") - == "access denied: the target file path is outside chroot" - ) - - def test_upload_limits_ok(self, mender_device_setup): - "File Transfer limits: file inside chroot; upload allowed" - - set_limits( - self.env, - self.mender_device, - { - "Enabled": True, - "FileTransfer": { - "Chroot": "/var/lib/mender/filetransfer", - "FollowSymLinks": True, # in the image /var/lib/mender is a symlink - }, - }, - self.auth, - self.devid, - ) - self.mender_device.run("mkdir -p /var/lib/mender/filetransfer") - - f = NamedTemporaryFile(delete=False) - f.write(os.urandom(16)) - f.close() - - fname = random_filename() - r = upload_file( - f"/var/lib/mender/filetransfer/{fname}", - open(f.name, "rb"), - self.devid, - self.auth_token, - ) - - assert r.status_code == 201 - - def test_upload_limits_err_file_too_big(self, mender_device_setup): - "File Transfer limits: file size over the limit; upload forbidden" - set_limits( - self.env, - self.mender_device, - { - "Enabled": True, - "FileTransfer": {"MaxFileSize": 15 * 1024, "FollowSymLinks": True}, - }, - self.auth, - self.devid, - ) - - f = NamedTemporaryFile(delete=False) - for i in range(128 * 1024): - f.write(b"ok") - f.close() - - fname = random_filename() - r = upload_file( - f"/tmp/{fname}", - open(f.name, "rb"), - self.devid, - self.auth_token, - ) - - assert r.status_code == 400, r.json() - assert ( - r.json().get("error") - == "failed to write file chunk: transmitted bytes limit exhausted" - ) - - def test_upload_limits_err_max_bytes_per_minute_exceeded(self, mender_device_setup): - "File Transfer limits: transfers during last minute over the limit; upload forbidden" - set_limits( - self.env, - self.mender_device, - { - "Enabled": True, - "FileTransfer": { - "FollowSymLinks": True, - "Counters": {"MaxBytesRxPerMinute": 16 * 1024}, - }, - }, - self.auth, - self.devid, - ) - - f = NamedTemporaryFile(delete=False) - for i in range(256 * 1024): - f.write(b"ok") - f.close() - - fname = random_filename() - upload_file( - f"/tmp/{fname}-0.bin", - open(f.name, "rb"), - self.devid, - self.auth_token, - ) - upload_file( - f"/tmp/{fname}-1.bin", - open(f.name, "rb"), - self.devid, - self.auth_token, - ) - logger.info("-- testcase: File Transfer limits: sleeping to gather the avg") - - time.sleep(60) # wait for mender-connect to calculate the 1m exp moving avg - self.mender_device.run( - "kill -USR1 `pidof mender-connect`" - ) # USR1 makes mender-connect print status - - r = upload_file( - f"/tmp/{fname}-2.bin", open(f.name, "rb"), self.devid, self.auth_token - ) - - assert r.status_code == 400, r.json() - assert r.json().get("error") == "transmitted bytes limit exhausted" - - logger.info( - "-- testcase: File Transfer limits: transfers during last minute: test_filetransfer_limits_upload sleeping 64s to be able to transfer again" - ) - # let's rest some more and increase the limit and try again - time.sleep(64) - self.mender_device.run( - "kill -USR1 `pidof mender-connect`" - ) # USR1 makes mender-connect print status - - logger.info( - "-- testcase: File Transfer limits: transfers during last minute below the limit; upload allowed" - ) - f = NamedTemporaryFile(delete=False) - for i in range(64): - f.write(b"ok") - f.close() - # upload a file - r = upload_file( - f"/tmp/{fname}-a.bin", - open(f.name, "rb"), - self.devid, - self.auth_token, - ) - self.mender_device.run( - "kill -USR1 `pidof mender-connect`" - ) # USR1 makes mender-connect print status - - assert r.status_code == 201 - - def test_upload_limits_err_preserve_modes(self, mender_device_setup): - "File Transfer limits: preserve modes;" - - set_limits( - self.env, - self.mender_device, - { - "Enabled": True, - "FileTransfer": { - "Chroot": "/var/lib/mender/filetransfer", - "FollowSymLinks": True, # in the image /var/lib/mender is a symlink - "PreserveMode": True, - }, - }, - self.auth, - self.devid, - ) - self.mender_device.run("mkdir -p /var/lib/mender/filetransfer") - - f = NamedTemporaryFile(delete=False) - f.write(os.urandom(16)) - f.close() - - fname = random_filename() - r = upload_file( - f"/var/lib/mender/filetransfer/{fname}.bin", - open(f.name, "rb"), - self.devid, - self.auth_token, - mode="4711", - ) - modes_ls = self.mender_device.run( - f"ls -al /var/lib/mender/filetransfer/{fname}.bin" - ) - logger.info( - f"test_filetransfer_limits_upload ls -al /var/lib/mender/filetransfer/{fname}.bin:\n%s" - % modes_ls - ) - - assert modes_ls.startswith("-rws--x--x") - assert r.status_code == 201 - - def test_upload_limits_preserve_owner_and_group(self, mender_device_setup): - "File Transfer limits: preserve owner and group;" - - set_limits( - self.env, - self.mender_device, - { - "Enabled": True, - "FileTransfer": { - "Chroot": "/var/lib/mender/filetransfer", - "FollowSymLinks": True, # in the image /var/lib/mender is a symlink - "PreserveOwner": True, - "PreserveGroup": True, - }, - }, - self.auth, - self.devid, - ) - self.mender_device.run("mkdir -p /var/lib/mender/filetransfer") - - f = NamedTemporaryFile(delete=False) - f.write(os.urandom(16)) - f.close() - gid = int(self.mender_device.run("cat /etc/group | tail -1 | cut -f3 -d:")) - uid = int(self.mender_device.run("cat /etc/passwd | tail -1 | cut -f3 -d:")) - logger.info("test_filetransfer_limits_upload gid/uid %d/%d", gid, uid) - fname = random_filename() - r = upload_file( - f"/var/lib/mender/filetransfer/{fname}.bin", - open(f.name, "rb"), - self.devid, - self.auth_token, - uid=str(uid), - gid=str(gid), - ) - - owner_group = self.mender_device.run( - f"ls -aln /var/lib/mender/filetransfer/{fname}.bin | cut -f 3,4 -d' '" - ) - - assert owner_group == str(uid) + " " + str(gid) + "\n" - assert r.status_code == 201 - - def assert_forbidden(self, rsp, message): - try: - assert rsp.status_code == 403 - assert rsp.json().get("error") == message - except AssertionError as e: - if rsp.status_code == 500: - raise NotImplementedError( - "[MEN-4659] Deviceconnect should not respond with 5xx errors " - + "on user restriction errors" - ) - else: - raise e - - @pytest.mark.xfail(raises=NotImplementedError, reason="MEN-4659") - def test_filetransfer_limits_download_err_outside_chroot(self, mender_device_setup): - "File Transfer limits: file outside chroot; download forbidden" - - set_limits( - self.env, - self.mender_device, - { - "Enabled": True, - "FileTransfer": {"Chroot": "/var/lib/mender/filetransfer"}, - }, - self.auth, - self.devid, - ) - - path = "/etc/profile" - r = download_file(path, self.devid, self.auth_token) - - self.assert_forbidden( - r, "access denied: the target file path is outside chroot" - ) - - @pytest.mark.xfail(raises=NotImplementedError, reason="MEN-4659") - def test_filetransfer_limits_download_err_max_file_size(self, mender_device_setup): - "File Transfer limits: file over the max file size limit; download forbidden" - set_limits( - self.env, - self.mender_device, - {"Enabled": True, "FileTransfer": {"MaxFileSize": 2}}, - self.auth, - self.devid, - ) - - path = "/etc/profile" - r = download_file(path, self.devid, self.auth_token) - - self.assert_forbidden(r, "access denied: the file size is over the limit") - - @pytest.mark.xfail(raises=NotImplementedError, reason="MEN-4659") - def test_filetransfer_limits_download_err_not_allowed_to_follow_link( - self, - mender_device_setup, - ): - "File Transfer limits: not allowed to follow a link; download forbidden" - set_limits( - self.env, - self.mender_device, - {"Enabled": True, "FileTransfer": {"FollowSymLinks": False}}, - self.auth, - self.devid, - ) - - fname = random_filename() - path = f"/tmp/{fname}-profile-link" - self.mender_device.run("ln -s /etc/profile " + path) - r = download_file(path, self.devid, self.auth_token) - - self.assert_forbidden(r, "access denied: forbidden to follow the link") - - @pytest.mark.xfail(raises=NotImplementedError, reason="MEN-4659") - def test_filetransfer_limits_download_err_not_allowed_to_follow_link_on_path_part( - self, - mender_device_setup, - ): - "File Transfer limits: not allowed to follow a link on path part; download forbidden" - - set_limits( - self.env, - self.mender_device, - {"Enabled": True, "FileTransfer": {"FollowSymLinks": False}}, - self.auth, - self.devid, - ) - - fname = random_filename() - self.mender_device.run(f"cd /tmp && mkdir {fname} && cd {fname} && ln -s /etc") - # now we have a link to the etc directory under /tmp/{fname}/etc - path = f"/tmp/{fname}/etc/profile" - r = download_file(path, self.devid, self.auth_token) - - self.assert_forbidden(r, "access denied: forbidden to follow the link") - - def test_filetransfer_limits_download_ok_allowed_to_follow_symlink( - self, - mender_device_setup, - ): - "File Transfer limits: not allowed to follow a link; download forbidden" - set_limits( - self.env, - self.mender_device, - {"Enabled": True, "FileTransfer": {"FollowSymLinks": True}}, - self.auth, - self.devid, - ) - - fname = random_filename() + ".symlink" - path = f"/tmp/{fname}" - self.mender_device.run("ln -s /etc/profile " + path) - - r = download_file(path, self.devid, self.auth_token) - - assert r.status_code == 200, r.json() - # in the mender docker client there is no PATH in the /etc/profile, but there is PS1= see QA-1527 - assert "PS1=" in str(r.content) - - @pytest.mark.xfail(raises=NotImplementedError, reason="MEN-4659") - def test_filetransfer_limits_download_err_owner_mismatch(self, mender_device_setup): - "File Transfer limits: file owner do not match; download forbidden" - - set_limits( - self.env, - self.mender_device, - {"Enabled": True, "FileTransfer": {"OwnerGet": ["someotheruser"]}}, - self.auth, - self.devid, - ) - - path = "/etc/profile" - r = download_file(path, self.devid, self.auth_token) - - self.assert_forbidden(r, "access denied: the file owner does not match") - - @pytest.mark.xfail(raises=NotImplementedError, reason="MEN-4659") - def test_filetransfer_limits_download_err_group_mismatch(self, mender_device_setup): - "File Transfer limits: file group do not match; download forbidden" - - set_limits( - self.env, - self.mender_device, - {"Enabled": True, "FileTransfer": {"GroupGet": ["someothergroup"]}}, - self.auth, - self.devid, - ) - - path = "/etc/profile" - r = download_file(path, self.devid, self.auth_token) - - self.assert_forbidden(r, "access denied: the file group does not match") - - def test_filetransfer_limits_download_err_not_a_regular_file( - self, mender_device_setup - ): - "File Transfer limits: file not a regular file; download forbidden" - set_limits( - self.env, - self.mender_device, - {"Enabled": True, "FileTransfer": {"RegularFilesOnly": True}}, - self.auth, - self.devid, - ) - - fname = random_filename() - path = f"/tmp/{fname}.fifo" - self.mender_device.run("mkfifo " + path) - r = download_file(path, self.devid, self.auth_token) - - assert r.status_code == 400, r.json() - assert r.json().get("error").endswith("path is not a regular file") - - def test_filetransfer_limits_download_ok_file_owner_match( - self, mender_device_setup - ): - "File Transfer limits: file owner match; download allowed" - set_limits( - self.env, - self.mender_device, - {"Enabled": True, "FileTransfer": {"OwnerGet": ["someotheruser", "root"]}}, - self.auth, - self.devid, - ) - - path = "/etc/profile" - r = download_file(path, self.devid, self.auth_token) - - assert r.status_code == 200, r.json() - # in the mender docker client there is no PATH in the /etc/profile, but there is PS1= see QA-1527 - assert "PS1=" in str(r.content) - - -def rerun_on_timeouts(err, *args): - if not issubclass(err[0], AssertionError): - return False - return "408" in str(err[1]) - - -@flaky(rerun_filter=rerun_on_timeouts) -class TestFileTransferDownloadOS(BaseTestFileTransferDownload): - """Tests the file transfer functionality""" - - @pytest.fixture(scope="function") - def mender_device_setup( - self, request, standard_setup_one_docker_client_bootstrapped - ): - env = standard_setup_one_docker_client_bootstrapped - - request.cls.auth = env.auth - request.cls.mender_device = env.device - request.cls.auth_token = env.auth.get_auth_token() - request.cls.env = env - - devices = devauth.get_devices_status("accepted") - assert 1 == len(devices) - request.cls.devid = devices[0]["id"] - - wait_for_connect(env.auth, request.cls.devid) - - def test_download_ok(self, mender_device_setup, content_assertion=None): - super().test_download_ok(mender_device_setup, content_assertion="ServerURL") - - -@flaky(rerun_filter=rerun_on_timeouts) -class TestFileTransferDownloadEnterprise(BaseTestFileTransferDownload): - """Tests the file transfer functionality for enterprise setup""" - - @pytest.fixture(scope="function") - def mender_device_setup(self, request, enterprise_one_docker_client_bootstrapped): - env = enterprise_one_docker_client_bootstrapped - devid, auth_token, auth, mender_device = prepare_env_for_connect( - env, - docker=True, - ignore_existing=True, - ) - request.cls.devid = devid - request.cls.auth_token = auth_token - request.cls.auth = auth - request.cls.mender_device = mender_device - request.cls.env = env - - -@flaky(rerun_filter=rerun_on_timeouts) -class TestFileTransferLimitsOS(BaseTestFileTransferLimits): - """Tests the file transfer functionality""" - - @pytest.fixture(scope="function") - def mender_device_setup( - self, request, standard_setup_one_docker_client_bootstrapped - ): - - env = standard_setup_one_docker_client_bootstrapped - - request.cls.auth = env.auth - request.cls.mender_device = env.device - request.cls.auth_token = env.auth.get_auth_token() - request.cls.env = env - - devices = devauth.get_devices_status("accepted") - assert 1 == len(devices) - request.cls.devid = devices[0]["id"] - - wait_for_connect(env.auth, request.cls.devid) - - -@flaky(rerun_filter=rerun_on_timeouts) -class TestFileTransferLimitsEnterprise(BaseTestFileTransferLimits): - """Tests the file transfer functionality for enterprise setup""" - - @pytest.fixture(scope="function") - def mender_device_setup(self, request, enterprise_one_docker_client_bootstrapped): - env = enterprise_one_docker_client_bootstrapped - devid, auth_token, auth, mender_device = prepare_env_for_connect( - env, - docker=True, - ignore_existing=True, - ) - request.cls.devid = devid - request.cls.auth_token = auth_token - request.cls.auth = auth - request.cls.mender_device = mender_device - request.cls.env = env diff --git a/tests/tests/test_grouping.py b/tests/tests/test_grouping.py deleted file mode 100644 index 5ddc80df2..000000000 --- a/tests/tests/test_grouping.py +++ /dev/null @@ -1,197 +0,0 @@ -# Copyright 2022 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ..common_setup import ( - standard_setup_two_clients_bootstrapped, - enterprise_two_clients_bootstrapped, -) -from .common_update import common_update_procedure -from ..MenderAPI import DeviceAuthV2, Deployments, Inventory, image, logger -from .mendertesting import MenderTesting -from ..helpers import Helpers - - -class BaseTestGrouping(MenderTesting): - def validate_group_responses(self, device_map, inv): - """Checks whether the device_map corresponds to the server's view of - the current groups, using all the possible ways to query for this. - device_map is a map of device to group.""" - - groups = [] - groups_map = {} - devices_with_group = [] - devices_without_group = [] - for device in device_map: - group = device_map[device] - if group is not None: - groups.append(group) - devices_with_group.append(device) - else: - devices_without_group.append(device) - - if groups_map.get(group): - groups_map[group].append(device) - else: - groups_map[group] = [device] - - assert sorted(inv.get_groups()) == sorted(groups) - assert sorted( - [device["id"] for device in inv.get_devices(has_group=True)] - ) == sorted(devices_with_group) - assert sorted( - [device["id"] for device in inv.get_devices(has_group=False)] - ) == sorted(devices_without_group) - assert sorted([device["id"] for device in inv.get_devices()]) == sorted( - device_map.keys() - ) - - for group in groups: - assert sorted(inv.get_devices_in_group(group)) == sorted(groups_map[group]) - for device in device_map: - assert inv.get_device_group(device)["group"] == device_map[device] - - def do_test_basic_groups(self, env): - """Tests various group operations.""" - inv = Inventory(env.auth) - - devices = [device["id"] for device in inv.get_devices()] - assert len(devices) == 2 - - # Purely for easier reading: Assign labels to each device. - alpha = devices[0] - bravo = devices[1] - - # Start out with no groups. - self.validate_group_responses({alpha: None, bravo: None}, inv) - - # Test various group operations. - inv.put_device_in_group(alpha, "Red") - self.validate_group_responses({alpha: "Red", bravo: None}, inv) - - inv.put_device_in_group(bravo, "Blue") - self.validate_group_responses({alpha: "Red", bravo: "Blue"}, inv) - - inv.delete_device_from_group(alpha, "Red") - self.validate_group_responses({alpha: None, bravo: "Blue"}, inv) - - # Note that this *moves* the device into the group. - inv.put_device_in_group(bravo, "Red") - self.validate_group_responses({alpha: None, bravo: "Red"}, inv) - - # Important: Leave the groups as you found them: Empty. - inv.delete_device_from_group(bravo, "Red") - self.validate_group_responses({alpha: None, bravo: None}, inv) - - def do_test_update_device_group(self, env, valid_image_with_mender_conf): - """ - Perform a successful upgrade on one group of devices, and assert that: - * deployment status/logs are correct. - * only the correct group is updated, not the other one. - - A reboot is performed, and running partitions have been swapped. - Deployment status will be set as successful for device. - Logs will not be retrieved, and result in 404. - """ - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - inv = Inventory(env.auth) - - # Beware that there will two parallel things going on below, one for - # each group. We aim to update the group alpha, not beta. - - mender_device_group = env.device_group - assert len(mender_device_group) == 2 - alpha = mender_device_group[0] - bravo = mender_device_group[1] - - ip_to_device_id = Helpers.ip_to_device_id_map( - mender_device_group, devauth=devauth - ) - id_alpha = ip_to_device_id[alpha.host_string] - id_bravo = ip_to_device_id[bravo.host_string] - logger.info("ID of alpha host: %s" % id_alpha) - logger.info("ID of bravo host: %s" % id_bravo) - - # TODO: parallelize these using fabric.group.ThreadingGroup once we upgrade to Python 3 - pass_part_alpha = alpha.get_passive_partition() - pass_part_bravo = bravo.get_passive_partition() - - inv.put_device_in_group(id_alpha, "Update") - - reboot = {alpha: None, bravo: None} - host_ip = env.get_virtual_network_host_ip() - with alpha.get_reboot_detector(host_ip) as reboot[ - alpha - ], bravo.get_reboot_detector(host_ip) as reboot[bravo]: - - mender_conf = alpha.run("cat /etc/mender/mender.conf") - deployment_id, expected_image_id = common_update_procedure( - valid_image_with_mender_conf(mender_conf), - devices=[id_alpha], - devauth=devauth, - deploy=deploy, - ) - - # Extra long wait here, because a real update takes quite a lot of time. - reboot[bravo].verify_reboot_not_performed(300) - reboot[alpha].verify_reboot_performed() - - assert alpha.get_passive_partition() != pass_part_alpha - assert bravo.get_passive_partition() == pass_part_bravo - - assert alpha.get_active_partition() == pass_part_alpha - assert bravo.get_active_partition() != pass_part_bravo - - deploy.check_expected_statistics( - deployment_id, expected_status="success", expected_count=1 - ) - - # No logs for either host: alpha because it was successful, bravo - # because it should never have attempted an update in the first place. - for id in [id_alpha, id_bravo]: - deploy.get_logs(id, deployment_id, expected_status=404) - - assert alpha.yocto_id_installed_on_machine() == expected_image_id - assert bravo.yocto_id_installed_on_machine() != expected_image_id - - # Important: Leave the groups as you found them: Empty. - inv.delete_device_from_group(id_alpha, "Update") - - -@MenderTesting.fast -class TestGroupingOpenSource(BaseTestGrouping): - def test_basic_groups(self, standard_setup_two_clients_bootstrapped): - self.do_test_basic_groups(standard_setup_two_clients_bootstrapped) - - def test_update_device_group( - self, standard_setup_two_clients_bootstrapped, valid_image_with_mender_conf - ): - self.do_test_update_device_group( - standard_setup_two_clients_bootstrapped, - valid_image_with_mender_conf, - ) - - -@MenderTesting.fast -class TestGroupingEnterprise(BaseTestGrouping): - def test_basic_groups(self, enterprise_two_clients_bootstrapped): - self.do_test_basic_groups(enterprise_two_clients_bootstrapped) - - def test_update_device_group( - self, enterprise_two_clients_bootstrapped, valid_image_with_mender_conf - ): - self.do_test_update_device_group( - enterprise_two_clients_bootstrapped, - valid_image_with_mender_conf, - ) diff --git a/tests/tests/test_image_update_failures.py b/tests/tests/test_image_update_failures.py deleted file mode 100644 index 3ba459d1b..000000000 --- a/tests/tests/test_image_update_failures.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2023 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ..common_setup import ( - standard_setup_one_client_bootstrapped, - enterprise_one_client_bootstrapped, -) -from .common_update import common_update_procedure -from .mendertesting import MenderTesting -from ..MenderAPI import DeviceAuthV2, Deployments - - -class BaseTestFailures(MenderTesting): - @MenderTesting.slow - def do_test_update_image_id_already_installed( - self, - env, - valid_image_with_mender_conf, - ): - """Test that an image with the same ID as the already installed image does not install anew""" - - mender_device = env.device - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - host_ip = env.get_virtual_network_host_ip() - with mender_device.get_reboot_detector(host_ip) as reboot: - mender_conf = mender_device.run("cat /etc/mender/mender.conf") - deployment_id, expected_image_id = common_update_procedure( - valid_image_with_mender_conf(mender_conf), - verify_status=True, - devauth=devauth, - deploy=deploy, - ) - reboot.verify_reboot_performed() - - devices_accepted_id = [ - device["id"] for device in devauth.get_devices_status("accepted") - ] - deployment_id = deploy.trigger_deployment( - name="New valid update", - artifact_name=expected_image_id, - devices=devices_accepted_id, - ) - - deploy.check_expected_statistics(deployment_id, "already-installed", 1) - deploy.check_expected_status("finished", deployment_id) - - @MenderTesting.fast - def do_test_large_update_image(self, env, large_image): - """Installing an image larger than the passive/active partition size should result in a failure.""" - - mender_device = env.device - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - host_ip = env.get_virtual_network_host_ip() - with mender_device.get_reboot_detector(host_ip) as reboot: - deployment_id, _ = common_update_procedure( - install_image=large_image, - # We use verify_status=False because the device is very quick in reporting - # failure and the test framework might miss the 'inprogress' status transition. - verify_status=False, - devauth=devauth, - deploy=deploy, - ) - deploy.check_expected_statistics(deployment_id, "failure", 1) - reboot.verify_reboot_not_performed() - deploy.check_expected_status("finished", deployment_id) - - -class TestFailuresOpenSource(BaseTestFailures): - @MenderTesting.slow - def test_update_image_id_already_installed( - self, - standard_setup_one_client_bootstrapped, - valid_image_with_mender_conf, - ): - self.do_test_update_image_id_already_installed( - standard_setup_one_client_bootstrapped, valid_image_with_mender_conf - ) - - @MenderTesting.fast - def test_large_update_image( - self, standard_setup_one_client_bootstrapped, large_image - ): - self.do_test_large_update_image( - standard_setup_one_client_bootstrapped, large_image - ) - - -class TestFailuresOpenEnterprise(BaseTestFailures): - @MenderTesting.slow - def test_update_image_id_already_installed( - self, - enterprise_one_client_bootstrapped, - valid_image_with_mender_conf, - ): - self.do_test_update_image_id_already_installed( - enterprise_one_client_bootstrapped, valid_image_with_mender_conf - ) - - @MenderTesting.fast - def test_large_update_image(self, enterprise_one_client_bootstrapped, large_image): - self.do_test_large_update_image(enterprise_one_client_bootstrapped, large_image) diff --git a/tests/tests/test_inventory.py b/tests/tests/test_inventory.py deleted file mode 100644 index 8e79a3b83..000000000 --- a/tests/tests/test_inventory.py +++ /dev/null @@ -1,316 +0,0 @@ -# Copyright 2022 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import pytest -import tempfile -import time - -from .. import conftest -from ..common_setup import ( - standard_setup_one_client_bootstrapped, - enterprise_one_client_bootstrapped, -) -from ..MenderAPI import DeviceAuthV2, Deployments, Inventory, logger -from ..helpers import Helpers -from .common_artifact import get_script_artifact -from .mendertesting import MenderTesting - - -def make_script_artifact(artifact_name, device_type, output_path, extra_args): - script = b"""\ -#!/bin/bash -exit 0 -""" - return get_script_artifact( - script, artifact_name, device_type, output_path, extra_args - ) - - -class BaseTestInventory(MenderTesting): - def do_test_inventory(self, env): - """ - Test that device reports inventory after having bootstrapped and performed - an application update using a dummy script artifact. - """ - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - inv = Inventory(env.auth) - - # Install the script update module required for this test - Helpers.install_community_update_module(env.device, "script") - - def deploy_simple_artifact(artifact_name, extra_args): - # create a simple artifact (script) which doesn't do anything - with tempfile.NamedTemporaryFile() as tf: - artifact = make_script_artifact( - artifact_name, - conftest.machine_name, - tf.name, - extra_args=extra_args, - ) - deploy.upload_image(artifact) - - # deploy the artifact above - device_ids = [device["id"] for device in devauth.get_devices()] - deployment_id = deploy.trigger_deployment( - artifact_name, - artifact_name=artifact_name, - devices=device_ids, - ) - - # now just wait for the update to succeed - deploy.check_expected_statistics(deployment_id, "success", 1) - deploy.check_expected_status("finished", deployment_id) - - deploy_simple_artifact( - "simple-artifact-1", - "--software-name swname --software-version v1" - + " --provides rootfs-image.swname.custom_field:value" - + " --provides rootfs-image.custom_field:value", - ) - deploy_simple_artifact( - "simple-artifact-2", "--software-name swname --software-version v2" - ) - - # verify the inventory - latest_exception = None - for _ in range(10): - try: - inv_json = inv.get_devices() - assert len(inv_json) > 0 - - auth_json = devauth.get_devices() - auth_ids = [device["id"] for device in auth_json] - - for device in inv_json: - try: - # Check that authentication and inventory agree. - assert device["id"] in auth_ids - attrs = device["attributes"] - - # Extract name and value only, to make tests more resilient - attrs = [ - {"name": x.get("name"), "value": x.get("value")} - for x in attrs - ] - - # Check individual attributes. - network_interfaces = [ - elem - for elem in attrs - if elem["name"] == "network_interfaces" - ] - assert len(network_interfaces) == 1 - network_interfaces = network_interfaces[0] - if type(network_interfaces["value"]) is str: - assert any( - network_interfaces["value"] == iface - for iface in ["eth0", "enp0s3"] - ) - else: - assert any( - iface in network_interfaces["value"] - for iface in ["eth0", "enp0s3"] - ) - assert ( - json.loads( - '{"name": "hostname", "value": "%s"}' - % conftest.machine_name - ) - in attrs - ) - assert ( - json.loads( - '{"name": "device_type", "value": "%s"}' - % conftest.machine_name - ) - in attrs - ) - # Should be in inventory because it comes with artifact. - assert ( - json.loads( - '{"name": "rootfs-image.swname.version", "value": "v2"}' - ) - in attrs - ) - # Should not be in inventory because the default is to - # clear inventory attributes in the same namespace. - assert ( - json.loads( - '{"name": "rootfs-image.swname.custom_field", "value": "value"}' - ) - not in attrs - ) - # Should be in inventory because the default is to keep - # inventory attributes in different namespaces. - assert ( - json.loads( - '{"name": "rootfs-image.custom_field", "value": "value"}' - ) - in attrs - ) - - if conftest.machine_name == "qemux86-64": - bootloader_integration = "uefi_grub" - elif conftest.machine_name == "vexpress-qemu": - bootloader_integration = "uboot" - else: - pytest.fail( - "Unknown machine_name. Please add an expected bootloader_integration for this machine_name" - ) - assert ( - json.loads( - '{"name": "mender_bootloader_integration", "value": "%s"}' - % bootloader_integration - ) - in attrs - ) - - # Check that all known keys are present. - keys = [str(attr["name"]) for attr in attrs] - expected_keys = [ - "hostname", - "network_interfaces", - "cpu_model", - "mem_total_kB", - "device_type", - [ - "ipv4_enp0s3", - "ipv6_enp0s3", - "ipv4_eth0", - ], # Multiple possibilities - ["mac_enp0s3", "mac_eth0"], - "mender_client_version", - "artifact_name", - "kernel", - "os", - "geo-ip", - "geo-city", - "geo-country", - "geo-lat", - "geo-lon", - "geo-timezone", - ] - for key in expected_keys: - if type(key) is list: - assert any([subkey in keys for subkey in key]) - else: - assert key in keys - except Exception as e: - logger.info( - f"Exception caught, 'device' json: {device}, exception {str(e)}" - ) - raise - except Exception as e: - latest_exception = e - time.sleep(5) - else: - return - raise latest_exception - - def do_test_inventory_update_after_successful_deployment(self, env): - """ - Test that device reports inventory after a new successful deployment, - and not simply after a boot. - """ - - mender_device = env.device - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - inv = Inventory(env.auth) - - # Give the image a larger wait interval. - sedcmd = "sed -i.bak 's/%s/%s/' /etc/mender/mender.conf" % ( - r"\(InventoryPollInter.*:\)\( *[0-9]*\)", - "\\1 300", - ) - mender_device.run(sedcmd) - mender_device.run("systemctl restart mender-updated") - - # Install the script update module required for this test - Helpers.install_community_update_module(mender_device, "script") - - # Get the inventory sent after first boot - initial_inv_json = inv.get_devices() - assert len(initial_inv_json) > 0 - assert "rootfs-image.swname.version" not in str( - initial_inv_json - ), "The initial inventory is not clean" - - def deploy_simple_artifact(artifact_name, extra_args): - # create a simple artifact (script) which doesn't do anything - with tempfile.NamedTemporaryFile() as tf: - artifact = make_script_artifact( - artifact_name, - conftest.machine_name, - tf.name, - extra_args=extra_args, - ) - deploy.upload_image(artifact) - - # deploy the artifact above - device_ids = [device["id"] for device in devauth.get_devices()] - deployment_id = deploy.trigger_deployment( - artifact_name, - artifact_name=artifact_name, - devices=device_ids, - ) - - # now just wait for the update to succeed - deploy.check_expected_statistics(deployment_id, "success", 1) - deploy.check_expected_status("finished", deployment_id) - - deploy_simple_artifact( - "simple-artifact-1", - "--software-name swname --software-version v1" - + " --provides rootfs-image.swname.custom_field:value", - ) - - # Give the client a little bit of time to do the update - time.sleep(15) - - post_deployment_inv_json = inv.get_devices() - assert len(post_deployment_inv_json) > 0 - assert "rootfs-image.swname.version" in str( - post_deployment_inv_json - ), "The device has not updated the inventory after the update" - - -class TestInventoryOpenSource(BaseTestInventory): - @MenderTesting.fast - def test_inventory(self, standard_setup_one_client_bootstrapped): - self.do_test_inventory(standard_setup_one_client_bootstrapped) - - @MenderTesting.fast - def test_inventory_update_after_successful_deployment( - self, standard_setup_one_client_bootstrapped - ): - self.do_test_inventory_update_after_successful_deployment( - standard_setup_one_client_bootstrapped - ) - - -class TestInventoryEnterprise(BaseTestInventory): - @MenderTesting.fast - def test_inventory(self, enterprise_one_client_bootstrapped): - self.do_test_inventory(enterprise_one_client_bootstrapped) - - @MenderTesting.fast - def test_inventory_update_after_successful_deployment( - self, enterprise_one_client_bootstrapped - ): - self.do_test_inventory_update_after_successful_deployment( - enterprise_one_client_bootstrapped - ) diff --git a/tests/tests/test_legacy_golang_update.py b/tests/tests/test_legacy_golang_update.py deleted file mode 100644 index 115ccb9fc..000000000 --- a/tests/tests/test_legacy_golang_update.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright 2023 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ..common_setup import ( - setup_with_legacy_v3_client, - enterprise_with_legacy_v3_client, -) -from .common_update import update_image, update_image_failed -from ..MenderAPI import DeviceAuthV2, Deployments, logger -from .mendertesting import MenderTesting - - -class BaseTestLegacyGolangUpdate(MenderTesting): - def do_test_migrate_from_legacy_mender_v3_success( - self, - env, - valid_image, - ): - """ - Start a legacy client (3.6 bundle, the last golang client) and do two successful updates. - The first one to validate 3.6 to latest upgrade and the following one to validate that - the updated device is fully capable. - """ - - mender_device = env.device - host_ip = env.get_virtual_network_host_ip() - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - update_image( - mender_device, - host_ip, - install_image=valid_image, - devauth=devauth, - deploy=deploy, - ) - - update_image( - mender_device, - host_ip, - install_image=valid_image, - devauth=devauth, - deploy=deploy, - ) - - def do_test_migrate_from_legacy_mender_v3_failure( - self, - env, - valid_image, - broken_update_image, - ): - """ - Start a legacy client (3.6 bundle, the last golang client) and do one failed update followed - but a successful one. - """ - - mender_device = env.device - host_ip = env.get_virtual_network_host_ip() - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - # Note that for the failed update, we still expect the v3 log message. See: - # https://github.com/mendersoftware/integration/commit/5971901131afbaf14cd4d9545d52f58366967fd5 - update_image_failed( - env.device, - env.get_virtual_network_host_ip(), - expected_log_message="Reboot to the new update failed", - devauth=devauth, - deploy=deploy, - ) - - update_image( - mender_device, - host_ip, - install_image=valid_image, - devauth=devauth, - deploy=deploy, - ) - - -class TestLegacyGolangUpdateOpenSource(BaseTestLegacyGolangUpdate): - def test_migrate_from_legacy_mender_v3_success( - self, setup_with_legacy_v3_client, valid_image - ): - self.do_test_migrate_from_legacy_mender_v3_success( - setup_with_legacy_v3_client, valid_image - ) - - def test_migrate_from_legacy_mender_v3_failure( - self, - setup_with_legacy_v3_client, - valid_image, - broken_update_image, - ): - self.do_test_migrate_from_legacy_mender_v3_failure( - setup_with_legacy_v3_client, - valid_image, - broken_update_image, - ) - - -class TestLegacyGolangUpdateEnterprise(BaseTestLegacyGolangUpdate): - def test_migrate_from_legacy_mender_v3_success( - self, enterprise_with_legacy_v3_client, valid_image_with_mender_conf - ): - mender_conf = enterprise_with_legacy_v3_client.device.run( - "cat /etc/mender/mender.conf" - ) - self.do_test_migrate_from_legacy_mender_v3_success( - enterprise_with_legacy_v3_client, valid_image_with_mender_conf(mender_conf) - ) - - def test_migrate_from_legacy_mender_v3_failure( - self, - enterprise_with_legacy_v3_client, - valid_image_with_mender_conf, - broken_update_image, - ): - mender_conf = enterprise_with_legacy_v3_client.device.run( - "cat /etc/mender/mender.conf" - ) - self.do_test_migrate_from_legacy_mender_v3_failure( - enterprise_with_legacy_v3_client, - valid_image_with_mender_conf(mender_conf), - broken_update_image, - ) diff --git a/tests/tests/test_monitor_client.py b/tests/tests/test_monitor_client.py deleted file mode 100644 index 6c7905499..000000000 --- a/tests/tests/test_monitor_client.py +++ /dev/null @@ -1,1920 +0,0 @@ -# Copyright 2023 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import datetime -import json -import os -import os.path -import pytest -import shutil -import tempfile -import time -import uuid -import inspect - -from email.parser import Parser -from email.policy import default -from redo import retriable -from ..common_setup import monitor_commercial_setup_no_client - -from ..MenderAPI import ( - authentication, - get_container_manager, - DeviceAuthV2, - DeviceMonitor, - Inventory, - logger, -) - -from testutils.api import useradm -from testutils.api.client import ApiClient -from testutils.infra.container_manager import factory -from testutils.infra import smtpd_mock -from testutils.common import User, new_tenant_client -from testutils.infra.cli import CliTenantadm - -container_factory = factory.get_factory() -connect_service_name = "mender-connect" - -# the smtpd saves messages in the following format: -# ---------- MESSAGE FOLLOWS ---------- -# b'Subject: it is fine' -# b'From: me@me.pl' -# b'To: local@local.pl' -# b'X-Peer: 127.0.0.1' -# b'' -# b'hej' -# b'' -# b':)' -# b'' -# b'local' -# ------------ END MESSAGE ------------ -message_start = "---------- MESSAGE FOLLOWS ----------" -message_end = "------------ END MESSAGE ------------" -message_mail_options_prefix = "mail options:" - -# tests constants -mailbox_path = "/var/spool/mail/local" -wait_for_alert_interval_s = 8 -alert_expiration_time_seconds = 32 -expected_from = "no-reply@hosted.mender.io" -daemon_main_loop_sleep_s = 2 - - -@retriable(sleeptime=60, attempts=5) -def get_and_parse_email_n(env, address, n): - mail, messages = get_and_parse_email(env, address) - assert len(messages) >= n - return mail, messages - - -def get_and_parse_email(env, address): - # get the email from the SMTP server - mail = env.get_file("local-smtp", mailbox_path) - logger.debug("got mail: '%s'", mail) - # read spool line by line, eval(line).decode('utf-8') for each line in lines - # between start and end of message - # concat and create header object for each - headers = [] - message_string = "" - device_date = None - for line in mail.splitlines(): - if line.startswith(message_mail_options_prefix): - continue - if message_start == line: - message_string = "" - device_date = None - continue - if message_end == line: - if device_date is not None: - logger.debug(f"using device_date {device_date}") - message_string = ( - device_date.strftime("Date: %a, %m %b %Y %H:%M:%S +0200") - + "\n" - + message_string - ) - logger.debug("msg:%s" % message_string) - h = Parser(policy=default).parsestr(message_string) - headers.append(h) - continue - # extra safety, we are supposed to only eval b'string' lines - if not line.startswith("b'"): - continue - line_string = eval(line).decode("utf-8") - message_string = message_string + line_string + "\n" - # get the date from b'Time on device: Thu, 01 Dec 2022 14:01:01 UTC' line - if line_string.startswith("Time on device: "): - device_date = datetime.datetime.strptime( - line_string, "Time on device: %a, %d %b %Y %H:%M:%S %Z" - ) - logger.debug(f"parsed device_date {device_date}") - - # log all messages for debug - for m in headers: - logger.debug("got message:") - logger.debug(" body: %s", m.get_body().get_content()) - logger.debug(" Bcc: %s", m["Bcc"]) - logger.debug(" From: %s", m["From"]) - logger.debug(" Subject: %s", m["Subject"]) - - return mail, headers - - -def assert_valid_alert(message, bcc, subject): - assert "Bcc" in message - assert "From" in message - assert "Subject" in message - assert message["Bcc"] == bcc - assert message["From"] == expected_from - assert message["Subject"].startswith(subject) - - -def prepare_service_monitoring(mender_device, service_name, use_ctl=False): - if use_ctl: - mender_device.run( - 'mender-monitorctl create service "%s" systemd' % (service_name) - ) - mender_device.run('mender-monitorctl enable service "%s"' % (service_name)) - else: - try: - monitor_available_dir = "/etc/mender-monitor/monitor.d/available" - monitor_enabled_dir = "/etc/mender-monitor/monitor.d/enabled" - mender_device.run("mkdir -p '%s'" % monitor_available_dir) - mender_device.run("mkdir -p '%s'" % monitor_enabled_dir) - mender_device.run("systemctl restart %s" % service_name) - tmpdir = tempfile.mkdtemp() - service_check_file = os.path.join(tmpdir, "service_" + service_name + ".sh") - f = open(service_check_file, "w") - f.write("SERVICE_NAME=%s\nSERVICE_TYPE=systemd\n" % service_name) - f.close() - mender_device.put( - os.path.basename(service_check_file), - local_path=os.path.dirname(service_check_file), - remote_path=monitor_available_dir, - ) - mender_device.run( - "ln -s '%s/service_%s.sh' '%s/service_%s.sh'" - % ( - monitor_available_dir, - service_name, - monitor_enabled_dir, - service_name, - ) - ) - finally: - shutil.rmtree(tmpdir) - - -def prepare_log_monitoring( - mender_device, - service_name, - log_file, - log_pattern, - log_pattern_expiration=None, - update_check_file_only=False, - use_ctl=False, -): - if use_ctl: - mender_device.run( - 'mender-monitorctl create log "%s" "%s" "%s"' - % (service_name, log_pattern, log_file) - ) - mender_device.run('mender-monitorctl enable log "%s"' % (service_name)) - else: - try: - monitor_available_dir = "/etc/mender-monitor/monitor.d/available" - monitor_enabled_dir = "/etc/mender-monitor/monitor.d/enabled" - if not update_check_file_only: - mender_device.run("mkdir -p '%s'" % monitor_available_dir) - mender_device.run("mkdir -p '%s'" % monitor_enabled_dir) - tmpdir = tempfile.mkdtemp() - service_check_file = os.path.join(tmpdir, "log_" + service_name + ".sh") - f = open(service_check_file, "w") - f.write( - 'SERVICE_NAME="%s"\nLOG_FILE="%s"\nLOG_PATTERN="%s"\n' - % (service_name, log_file, log_pattern) - ) - if log_pattern_expiration: - f.write("LOG_PATTERN_EXPIRATION=%d\n" % log_pattern_expiration) - f.close() - mender_device.put( - os.path.basename(service_check_file), - local_path=os.path.dirname(service_check_file), - remote_path=monitor_available_dir, - ) - if not update_check_file_only: - mender_device.run( - "ln -s '%s/log_%s.sh' '%s/log_%s.sh'" - % ( - monitor_available_dir, - service_name, - monitor_enabled_dir, - service_name, - ) - ) - finally: - shutil.rmtree(tmpdir) - - -def prepare_dbus_monitoring( - mender_device, dbus_name, log_pattern=None, dbus_pattern=None -): - assert not (log_pattern is None and dbus_pattern is None) - try: - monitor_available_dir = "/etc/mender-monitor/monitor.d/available" - monitor_enabled_dir = "/etc/mender-monitor/monitor.d/enabled" - mender_device.run("mkdir -p '%s'" % monitor_available_dir) - mender_device.run("mkdir -p '%s'" % monitor_enabled_dir) - tmpdir = tempfile.mkdtemp() - dbus_check_file = os.path.join(tmpdir, "dbus_test.sh") - f = open(dbus_check_file, "w") - f.write("DBUS_NAME=%s\n" % dbus_name) - if log_pattern: - f.write("DBUS_PATTERN=%s\n" % log_pattern) - if dbus_pattern: - f.write("DBUS_WATCH_EXPRESSION=%s\n" % dbus_pattern) - f.close() - mender_device.put( - os.path.basename(dbus_check_file), - local_path=os.path.dirname(dbus_check_file), - remote_path=monitor_available_dir, - ) - mender_device.run( - "ln -s '%s/dbus_test.sh' '%s/dbus_test.sh'" - % (monitor_available_dir, monitor_enabled_dir) - ) - # Give some time for the new monitor to be enabled - time.sleep(daemon_main_loop_sleep_s + 1) - finally: - shutil.rmtree(tmpdir) - - -def prepare_dockerevents_monitoring( - mender_device, container_name, alert_expiration="", action="restart" -): - mender_device.run( - "mender-monitorctl create dockerevents container_%s_%s %s %s %s" - % (container_name, action, container_name, action, alert_expiration) - ) - mender_device.run( - "mender-monitorctl enable dockerevents container_%s_%s" - % (container_name, action) - ) - - -class TestMonitorClientEnterprise: - """Tests for the Monitor client""" - - def prepare_env(self, env, user_name): - u = User("", user_name, "whatsupdoc") - cli = CliTenantadm(containers_namespace=env.name) - - uuidv4 = str(uuid.uuid4()) - name = "test.mender.io-" + uuidv4 - tid = cli.create_org(name, u.name, u.pwd, plan="enterprise", addons=["monitor"]) - - # at the moment we do not have a notion of a monitor add-on in the - # backend, but this will be needed here, see MEN-4809 - # update_tenant( - # tid, addons=["monitor"], container_manager=get_container_manager(), - # ) - - tenant = cli.get_tenant(tid) - tenant = json.loads(tenant) - - auth = authentication.Authentication(name=name, username=u.name, password=u.pwd) - auth.create_org = False - auth.reset_auth_token() - devauth_tenant = DeviceAuthV2(auth) - - mender_device = new_tenant_client(env, "mender-client", tenant["tenant_token"]) - mender_device.ssh_is_opened() - - devauth_tenant.accept_devices(1) - - devices = list( - set( - [ - device["id"] - for device in devauth_tenant.get_devices_status("accepted") - ] - ) - ) - assert 1 == len(devices) - - devid = devices[0] - authtoken = auth.get_auth_token() - - logger.info("%s: env ready.", inspect.stack()[1].function) - - return devid, authtoken, auth, mender_device - - def get_alerts_and_alert_count_for_device(self, inventory, devid): - r = inventory.get_device(devid) - assert r.status_code == 200 - inventory_data = r.json() - alert_count = alerts = None - for inventory_item in inventory_data["attributes"]: - if ( - inventory_item["scope"] == "monitor" - and inventory_item["name"] == "alert_count" - ): - alert_count = inventory_item["value"] - elif ( - inventory_item["scope"] == "monitor" - and inventory_item["name"] == "alerts" - ): - alerts = inventory_item["value"] - return alerts, alert_count - - def test_monitorclient_alert_email(self, monitor_commercial_setup_no_client): - """Tests the monitor client email alerting""" - service_name = "crond" - user_name = "ci.email.tests+{}@mender.io".format(str(uuid.uuid4())) - devid, _, auth, mender_device = self.prepare_env( - monitor_commercial_setup_no_client, user_name - ) - inventory = Inventory(auth) - - logger.info( - "test_monitorclient_alert_email: email alert on systemd service not running scenario." - ) - prepare_service_monitoring(mender_device, service_name) - time.sleep(2 * wait_for_alert_interval_s) - - mender_device.run("systemctl stop %s" % service_name) - logger.info( - "Stopped %s, sleeping %ds." % (service_name, wait_for_alert_interval_s) - ) - time.sleep(2 * wait_for_alert_interval_s) - - alerts, alert_count = self.get_alerts_and_alert_count_for_device( - inventory, devid - ) - assert (True, 1) == (alerts, alert_count) - - mail, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, 1 - ) - assert len(messages) > 0 - assert_valid_alert( - messages[0], - user_name, - "CRITICAL: Monitor Alert for Service " - + service_name - + " not running on " - + devid, - ) - assert "${workflow.input." not in mail - logger.info("test_monitorclient_alert_email: got CRITICAL alert email.") - - mender_device.run("systemctl start %s" % service_name) - logger.info( - "Started %s, sleeping %ds" % (service_name, wait_for_alert_interval_s) - ) - time.sleep(2 * wait_for_alert_interval_s) - - alerts, alert_count = self.get_alerts_and_alert_count_for_device( - inventory, devid - ) - assert (False, 0) == (alerts, alert_count) - - mail, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, 2 - ) - messages_count = len(messages) - assert messages_count >= 2 - assert_valid_alert( - messages[1], - user_name, - "OK: Monitor Alert for Service " - + service_name - + " not running on " - + devid, - ) - assert "${workflow.input" not in mail - logger.info("test_monitorclient_alert_email: got OK alert email.") - - logger.info( - "test_monitorclient_alert_email: email alert on log file containing a pattern scenario." - ) - log_file = "/tmp/mylog.log" - log_pattern = "session opened for user [a-z]*" - - service_name = "mylog" - prepare_log_monitoring( - mender_device, - service_name, - log_file, - log_pattern, - ) - time.sleep(2 * wait_for_alert_interval_s) - mender_device.run("echo 'some line 1' >> " + log_file) - mender_device.run("echo 'some line 2' >> " + log_file) - - time.sleep(2 * wait_for_alert_interval_s) - _, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, messages_count - ) - assert messages_count == len(messages) - - mender_device.run( - "echo -ne 'a new session opened for user root now\nsome line 3\n' >> " - + log_file - ) - - time.sleep(wait_for_alert_interval_s) - mender_device.run("echo 'some line 4' >> " + log_file) - mender_device.run("echo 'some line 5' >> " + log_file) - - time.sleep(2 * wait_for_alert_interval_s) - _, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, messages_count + 1 - ) - messages_count = len(messages) - assert_valid_alert( - messages[-1], - user_name, - "CRITICAL: Monitor Alert for Log file contains " - + log_pattern - + " on " - + devid, - ) - - pattern_expiration_seconds = 32 - logger.info( - "test_monitorclient_alert_email: CRITICAL received; setting pattern expiration time=%ds and waiting.", - pattern_expiration_seconds, - ) - prepare_log_monitoring( - mender_device, - service_name, - log_file, - log_pattern, - log_pattern_expiration=pattern_expiration_seconds, - update_check_file_only=True, - ) - time.sleep(2 * wait_for_alert_interval_s) - mail, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, messages_count + 1 - ) - messages_count = len(messages) - assert_valid_alert( - messages[-1], - user_name, - "OK: Monitor Alert for Log file contains " + log_pattern + " on " + devid, - ) - # in each CRITICAL and OK email we expect the pattern to be present at least 3 times - assert mail.count(log_pattern) >= 6 - # in each CRITICAL and OK email we expect the log path to be present at least 1 time - assert mail.count(log_pattern) >= 2 - logger.info( - "test_monitorclient_alert_email: got OK alert email after log pattern expiration." - ) - device_monitor = DeviceMonitor(auth) - alerts = device_monitor.get_alerts(devid) - assert len(alerts) > 1 - assert "subject" in alerts[1] - assert "details" in alerts[1]["subject"] - assert "line_matching" in alerts[1]["subject"]["details"] - assert "data" in alerts[1]["subject"]["details"]["line_matching"] - assert ( - "a new session opened for user root now" - == alerts[1]["subject"]["details"]["line_matching"]["data"] - ) - assert "subject" in alerts[1] - assert "details" in alerts[1]["subject"] - assert "lines_before" in alerts[1]["subject"]["details"] - assert len(alerts[1]["subject"]["details"]["lines_before"]) == 2 - assert len(alerts[1]["subject"]["details"]["lines_after"]) == 1 - assert "data" in alerts[1]["subject"]["details"]["lines_before"][0] - assert "data" in alerts[1]["subject"]["details"]["lines_before"][1] - assert ( - "some line 1" == alerts[1]["subject"]["details"]["lines_before"][0]["data"] - ) - assert ( - "some line 2" == alerts[1]["subject"]["details"]["lines_before"][1]["data"] - ) - assert "data" in alerts[1]["subject"]["details"]["lines_after"][0] - assert ( - "some line 3" == alerts[1]["subject"]["details"]["lines_after"][0]["data"] - ) - logger.debug( - "test_monitorclient_alert_email: got %s alerts" % json.dumps(alerts) - ) - logger.debug( - "test_monitorclient_alert_email: got line -B1: '%s' from alerts" - % alerts[1]["subject"]["details"]["lines_before"][0]["data"] - ) - logger.debug( - "test_monitorclient_alert_email: got line -B2: '%s' from alerts" - % alerts[1]["subject"]["details"]["lines_before"][1]["data"] - ) - logger.debug( - "test_monitorclient_alert_email: got line -A1: '%s' from alerts" - % alerts[1]["subject"]["details"]["lines_after"][0]["data"] - ) - - logger.info( - "test_monitorclient_alert_email: email alert a pattern found in the journalctl output scenario." - ) - service_name = "crond" - prepare_log_monitoring( - mender_device, - service_name, - "@journalctl -f -u " + service_name, - ": Started .*", - ) - mender_device.run("echo -ne > /tmp/mylog.log") - mender_device.run("systemctl restart mender-monitor") - time.sleep(wait_for_alert_interval_s) - mail, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, messages_count + 1 - ) - messages_count = len(messages) - assert_valid_alert( - messages[-1], - user_name, - "CRITICAL: Monitor Alert for Log file contains : Started", - ) - assert "${workflow.input" not in mail - - logger.info( - "test_monitorclient_alert_email: email alert on streaming logs pattern expiration scenario." - ) - prepare_log_monitoring( - mender_device, - service_name, - "@tail -f " + log_file, - log_pattern, - log_pattern_expiration=pattern_expiration_seconds, - update_check_file_only=True, - ) - mender_device.run("systemctl restart mender-monitor") - mender_device.run( - "echo -ne 'another line\na new session opened for user root now\nsome line 3\n' >> " - + log_file - ) - - logger.info( - "test_monitorclient_alert_email: '@tail -f logfile' scenario, waiting %ds for pattern to expire." - % (2 * pattern_expiration_seconds) - ) - time.sleep(2 * pattern_expiration_seconds) - _, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, messages_count + 1 - ) - assert_valid_alert( - messages[-1], - user_name, - "OK: Monitor Alert for Log file contains " + log_pattern + " on " + devid, - ) - logger.info( - "test_monitorclient_alert_email: got OK alert email after log pattern expiration in case of streaming log file." - ) - - def test_monitorclient_alert_email_with_group( - self, monitor_commercial_setup_no_client - ): - """Tests the monitor client email alerting with device in a static group""" - service_name = "crond" - user_name = "ci.email.tests+{}@mender.io".format(str(uuid.uuid4())) - devid, _, auth, mender_device = self.prepare_env( - monitor_commercial_setup_no_client, user_name - ) - device_static_group = "localGroup0" - inventory = Inventory(auth) - inventory.put_device_in_group(devid, device_static_group) - - logger.info( - "test_monitorclient_alert_email_with_group: email alert on systemd service not running scenario." - ) - prepare_service_monitoring(mender_device, service_name) - time.sleep(2 * wait_for_alert_interval_s) - - mender_device.run("systemctl stop %s" % service_name) - logger.info( - "Stopped %s, sleeping %ds." % (service_name, wait_for_alert_interval_s) - ) - time.sleep(2 * wait_for_alert_interval_s) - - alerts, alert_count = self.get_alerts_and_alert_count_for_device( - inventory, devid - ) - assert (True, 1) == (alerts, alert_count) - - mail, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, 1 - ) - assert len(messages) > 0 - assert_valid_alert( - messages[0], - user_name, - "CRITICAL: Monitor Alert for Service " - + service_name - + " not running on " - + devid - + " in group: " - + device_static_group, - ) - assert "${workflow.input." not in mail - logger.info( - "test_monitorclient_alert_email_with_group: got CRITICAL alert email." - ) - - mender_device.run("systemctl start %s" % service_name) - logger.info( - "Started %s, sleeping %ds" % (service_name, wait_for_alert_interval_s) - ) - time.sleep(2 * wait_for_alert_interval_s) - - alerts, alert_count = self.get_alerts_and_alert_count_for_device( - inventory, devid - ) - assert (False, 0) == (alerts, alert_count) - - mail, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, 2 - ) - messages_count = len(messages) - assert messages_count >= 2 - assert_valid_alert( - messages[1], - user_name, - "OK: Monitor Alert for Service " - + service_name - + " not running on " - + devid - + " in group: " - + device_static_group, - ) - assert "${workflow.input" not in mail - logger.info("test_monitorclient_alert_email_with_group: got OK alert email.") - - def test_monitorclient_flapping(self, monitor_commercial_setup_no_client): - """Tests the monitor client flapping support""" - wait_for_alert_interval_s = 120 - service_name = "crond" - user_name = "ci.email.tests+{}@mender.io".format(str(uuid.uuid4())) - devid, _, _, mender_device = self.prepare_env( - monitor_commercial_setup_no_client, user_name - ) - - prepare_service_monitoring(mender_device, service_name) - - max_start_stop_iterations = 32 - not_running_time = 2.2 - logger.info( - "test_monitorclient_flapping: running stop/start for %s, %d iterations, sleep in-between: %.1fs" - % (service_name, max_start_stop_iterations, not_running_time) - ) - while max_start_stop_iterations > 0: - max_start_stop_iterations = max_start_stop_iterations - 1 - mender_device.run("systemctl stop %s" % service_name) - time.sleep(not_running_time) - mender_device.run("systemctl start %s" % service_name) - time.sleep(not_running_time) - - logger.info( - "test_monitorclient_flapping: waiting for %s seconds" - % (2 * wait_for_alert_interval_s) - ) - time.sleep(2 * wait_for_alert_interval_s) - mail, messages = get_and_parse_email( - monitor_commercial_setup_no_client, user_name - ) - messages_flipping = list( - filter(lambda x: "going up and down" in x["Subject"], messages) - ) - assert len(messages_flipping) >= 1 - messages_count_flapping = messages.index(messages_flipping[-1]) + 1 - assert_valid_alert( - messages_flipping[-1], - user_name, - "CRITICAL: Monitor Alert for Service " - + service_name - + " going up and down on " - + devid, - ) - assert "${workflow.input" not in mail - logger.info("test_monitorclient_flapping: got CRITICAL alert email.") - - logger.info( - "test_monitorclient_flapping: waiting for %s seconds" - % wait_for_alert_interval_s - ) - - time.sleep(2 * wait_for_alert_interval_s) - mail, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, - user_name, - messages_count_flapping + 1, - ) - assert messages_count_flapping + 1 <= len(messages) - assert_valid_alert( - messages[-1], - user_name, - "OK: Monitor Alert for Service " - + service_name - + " not running on " - + devid, - ) - assert "${workflow.input" not in mail - logger.info("test_monitorclient_flapping: got OK alert email.") - - def test_monitorclient_alert_email_rbac(self, monitor_commercial_setup_no_client): - """Tests the monitor client email alerting respecting RBAC""" - # first let's get the OK and CRITICAL email alerts {{{ - service_name = "crond" - user_name = "ci.email.tests+{}@mender.io".format(str(uuid.uuid4())) - devid, _, auth, mender_device = self.prepare_env( - monitor_commercial_setup_no_client, user_name - ) - inventory = Inventory(auth) - - prepare_service_monitoring(mender_device, service_name) - time.sleep(2 * wait_for_alert_interval_s) - - mender_device.run("systemctl stop %s" % service_name) - logger.info( - "Stopped %s, sleeping %ds." % (service_name, wait_for_alert_interval_s) - ) - time.sleep(2 * wait_for_alert_interval_s) - - alerts, alert_count = self.get_alerts_and_alert_count_for_device( - inventory, devid - ) - assert (True, 1) == (alerts, alert_count) - - mail, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, 1 - ) - assert len(messages) > 0 - - assert_valid_alert( - messages[0], - user_name, - "CRITICAL: Monitor Alert for Service " - + service_name - + " not running on " - + devid, - ) - assert "${workflow.input" not in mail - logger.info("test_monitorclient_alert_email_rbac: got CRITICAL alert email.") - - mender_device.run("systemctl start %s" % service_name) - logger.info( - "Started %s, sleeping %ds" % (service_name, wait_for_alert_interval_s) - ) - time.sleep(2 * wait_for_alert_interval_s) - - alerts, alert_count = self.get_alerts_and_alert_count_for_device( - inventory, devid - ) - assert (False, 0) == (alerts, alert_count) - - mail, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, 2 - ) - messages_count = len(messages) - assert messages_count >= 2 - assert_valid_alert( - messages[1], - user_name, - "OK: Monitor Alert for Service " - + service_name - + " not running on " - + devid, - ) - assert "${workflow.input" not in mail - logger.info("test_monitorclient_alert_email_rbac: got OK alert email.") - # }}} we got the CRITICAL and OK emails - - # let's add a role, that will allow user to view only devices of given group {{{ - uadm = ApiClient( - host=get_container_manager().get_mender_gateway(), - base_url=useradm.URL_MGMT, - ) - - role = { - "name": "deviceaccess", - "permissions": [ - { - "action": "VIEW_DEVICE", - "object": {"type": "DEVICE_GROUP", "value": "fullTestDevices"}, - } - ], - } - res = uadm.call( - "POST", useradm.URL_ROLES, headers=auth.get_auth_token(), body=role - ) - assert res.status_code == 201 - logger.info( - "test_monitorclient_alert_email_rbac: added role: restrict access to a group." - ) - # }}} role added - - # let's set the role for the user {{{ - res = uadm.call("GET", useradm.URL_USERS, headers=auth.get_auth_token()) - assert res.status_code == 200 - logger.info( - "test_monitorclient_alert_email_rbac: " - "get users: http rc: %d; response body: '%s'; " - % (res.status_code, res.json()) - ) - users = res.json() - res = uadm.call( - "PUT", - useradm.URL_USERS_ID.format(id=users[0]["id"]), - headers=auth.get_auth_token(), - body={"roles": ["deviceaccess"]}, - ) - assert res.status_code == 204 - logger.info("test_monitorclient_alert_email_rbac: role assigned to user.") - # }}} user has access only to fullTestDevices group - - # let's stop the service by name=service_name - mender_device.run("systemctl stop %s" % service_name) - logger.info( - "Stopped %s, sleeping %ds." % (service_name, wait_for_alert_interval_s) - ) - - time.sleep(2 * wait_for_alert_interval_s) - _, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, messages_count - ) - assert len(messages) == messages_count - # we did not receive any email -- user has no access to the device - logger.info( - "test_monitorclient_alert_email_rbac: did not receive CRITICAL email alert." - ) - - mender_device.run("systemctl start %s" % service_name) - logger.info( - "Started %s, sleeping %ds" % (service_name, wait_for_alert_interval_s) - ) - - time.sleep(2 * wait_for_alert_interval_s) - _, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, messages_count - ) - assert len(messages) == messages_count - # we did not receive any email -- user has no access to the device - logger.info( - "test_monitorclient_alert_email_rbac: did not receive OK email alert." - ) - - def test_monitorclient_alert_store(self, monitor_commercial_setup_no_client): - """Tests the monitor client alert local store""" - service_name = "rpcbind" - hostname = os.environ.get("GATEWAY_HOSTNAME", "docker.mender.io") - user_name = "ci.email.tests+{}@mender.io".format(str(uuid.uuid4())) - devid, _, _, mender_device = self.prepare_env( - monitor_commercial_setup_no_client, user_name - ) - - logger.info( - "test_monitorclient_alert_store: store alerts when offline scenario." - ) - - prepare_service_monitoring(mender_device, service_name) - time.sleep(2 * wait_for_alert_interval_s) - - logger.info( - "test_monitorclient_alert_store: disabling access to %s (point to localhost in /etc/hosts)" - % hostname - ) - mender_device.run("sed -i.backup -e '$a127.2.0.1 %s' /etc/hosts" % hostname) - mender_device.run("systemctl restart mender-authd") - mender_device.run("systemctl stop %s" % service_name) - logger.info( - "Stopped %s, sleeping %ds." % (service_name, wait_for_alert_interval_s) - ) - expected_alerts_count = 1 # one for CRITICAL, because we stopped the service - time.sleep(wait_for_alert_interval_s) - mender_device.run("systemctl start %s" % service_name) - logger.info( - "Started %s, sleeping %ds" % (service_name, wait_for_alert_interval_s) - ) - expected_alerts_count = ( - expected_alerts_count + 1 - ) # one for OK, because we started the service - - time.sleep(2 * wait_for_alert_interval_s) - _, messages = get_and_parse_email(monitor_commercial_setup_no_client, user_name) - assert len(messages) == 0 - logger.info("test_monitorclient_alert_store: got no alerts, device is offline.") - - logger.info( - "test_monitorclient_alert_store: re-enabling access to %s (restoring /etc/hosts)" - % hostname - ) - mender_device.run("mv /etc/hosts.backup /etc/hosts") - logger.info("test_monitorclient_alert_store: waiting for alerts to come.") - - time.sleep(8 * wait_for_alert_interval_s) - mail, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, 2 - ) - logger.info("got %d alert messages", len(messages)) - - assert len(messages) > 1 - - # Sort emails by date - messages.sort(key=lambda x: x["Date"]) - - assert_valid_alert( - messages[0], - user_name, - "CRITICAL: Monitor Alert for Service " - + service_name - + " not running on " - + devid, - ) - assert "${workflow.input" not in mail - logger.info("test_monitorclient_alert_store: got CRITICAL alert email.") - - assert_valid_alert( - messages[1], - user_name, - "OK: Monitor Alert for Service " - + service_name - + " not running on " - + devid, - ) - assert "${workflow.input" not in mail - logger.info("test_monitorclient_alert_store: got OK alert email.") - - logger.info("test_monitorclient_alert_store: large alert store (MEN-5133).") - log_file = "/tmp/mylog.log" - log_pattern = "session opened for user [a-z]*" - - service_name = "mylog" - prepare_log_monitoring( - mender_device, - service_name, - log_file, - log_pattern, - ) - mender_device.run("touch '" + log_file + "'") - logger.info( - "test_monitorclient_alert_store: large store disabling access to %s (point to localhost in /etc/hosts)" - % hostname - ) - mender_device.run("sed -i.backup -e '$a127.2.0.1 %s' /etc/hosts" % hostname) - mender_device.run("systemctl restart mender-authd") - - patterns_count = 30 - expected_alerts_count = ( - expected_alerts_count + 1 - ) # one for all the patterns detected in log (changed with MEN-5458) - for i in range(patterns_count): - mender_device.run( - "for i in {1..4}; do echo 'some line '$i >> " + log_file + "; done" - ) - mender_device.run( - "echo 'the session session opened for user tests' >> " + log_file - ) - mender_device.run( - "for i in {6..9}; do echo 'some line '$i >> " + log_file + "; done" - ) - time.sleep(wait_for_alert_interval_s) - - logger.info( - "test_monitorclient_alert_store: re-enabling access to %s (restoring /etc/hosts)" - % hostname - ) - mender_device.run("mv /etc/hosts.backup /etc/hosts") - time.sleep( - 9 * wait_for_alert_interval_s - ) # at the moment we send stored alerts every minute - - _, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, expected_alerts_count - ) - assert len(messages) >= expected_alerts_count - logger.info("got %d alert messages." % len(messages)) - - def test_dbus_pattern_match(self, monitor_commercial_setup_no_client): - """Test the dbus subsystem""" - dbus_name = "test" - user_name = "ci.email.tests+{}@mender.io".format(str(uuid.uuid4())) - devid, _, _, mender_device = self.prepare_env( - monitor_commercial_setup_no_client, user_name - ) - - logger.info( - "test_dbus_pattern_match: email alert on dbus signal pattern match scenario." - ) - prepare_dbus_monitoring( - mender_device, dbus_name, log_pattern="JwtTokenStateChange" - ) - - # Monitoring with only DBUS_PATTERN is buggy and the alert is sometimes missed. - # See test_dbus_bus_filter for comparison, where filtering with DBUS_WATCH_EXPRESSION - # reliably sends an alert with a single trigger. - # Try in a loop for the same amount of time that get_and_parse_email_n would retry - tries_left = 10 - while tries_left > 0: - # Call FetchJwtToken to trigger the signal (string "JwtTokenStateChange") - mender_device.run( - "dbus-send --system --dest=io.mender.AuthenticationManager /io/mender/AuthenticationManager io.mender.Authentication1.FetchJwtToken" - ) - mail, messages = get_and_parse_email( - monitor_commercial_setup_no_client, user_name - ) - if len(messages) > 0: - break - logger.info("test_dbus_pattern_match: got no alerts, retrying in 30s") - time.sleep(30) - tries_left = tries_left - 1 - else: - pytest.fail("Did not receive any messages") - - assert_valid_alert( - messages[0], - user_name, - "CRITICAL: Monitor Alert for D-Bus signal arrived on bus system bus on " - + devid, - ) - assert "${workflow.input" not in mail - logger.info("test_dbus_pattern_match: got CRITICAL alert email.") - - def test_dbus_bus_filter(self, monitor_commercial_setup_no_client): - """Test the dbus subsystem""" - dbus_name = "test" - user_name = "ci.email.tests+{}@mender.io".format(str(uuid.uuid4())) - devid, _, _, mender_device = self.prepare_env( - monitor_commercial_setup_no_client, user_name - ) - - logger.info( - "test_dbus_bus_filter: email alert on single dbus filter signal scenario." - ) - prepare_dbus_monitoring( - mender_device, - dbus_name, - dbus_pattern="type='signal',interface='io.mender.Authentication1'", - ) - - # Call FetchJwtToken to trigger the signal - mender_device.run( - "dbus-send --system --dest=io.mender.AuthenticationManager /io/mender/AuthenticationManager io.mender.Authentication1.FetchJwtToken" - ) - - time.sleep(2 * wait_for_alert_interval_s) - mail, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, 1 - ) - assert len(messages) > 0 - assert_valid_alert( - messages[0], - user_name, - "CRITICAL: Monitor Alert for D-Bus signal arrived on bus type=signal,interface=io.mender.Authentication1 on " - + devid, - ) - assert "${workflow.input" not in mail - logger.info("test_dbus_bus_filter: got CRITICAL alert email.") - - def test_monitorclient_logs_and_services(self, monitor_commercial_setup_no_client): - """Tests the monitor client email alerting for multiple services with extra checks""" - user_name = "ci.email.tests+{}@mender.io".format(str(uuid.uuid4())) - devid, _, auth, mender_device = self.prepare_env( - monitor_commercial_setup_no_client, user_name - ) - inventory = Inventory(auth) - - logger.info( - "test_monitorclient_logs_and_services: email alert on systemd service not running scenario." - ) - for service_name in ["crond", "mender-connect"]: - prepare_service_monitoring(mender_device, service_name, use_ctl=True) - - time.sleep(2 * wait_for_alert_interval_s) - - for service_name in ["crond", "mender-connect"]: - mender_device.run("systemctl stop %s" % service_name) - logger.info( - "Stopped %s, sleeping %ds." % (service_name, wait_for_alert_interval_s) - ) - time.sleep(wait_for_alert_interval_s) - - alerts, alert_count = self.get_alerts_and_alert_count_for_device( - inventory, devid - ) - assert (True, 2) == (alerts, alert_count) - - for service_name in ["crond", "mender-connect"]: - mender_device.run("systemctl start %s" % service_name) - logger.info( - "Started %s, sleeping %ds" % (service_name, wait_for_alert_interval_s) - ) - time.sleep(wait_for_alert_interval_s) - - alerts, alert_count = self.get_alerts_and_alert_count_for_device( - inventory, devid - ) - assert (False, 0) == (alerts, alert_count) - - time.sleep(2 * wait_for_alert_interval_s) - mail, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, 4 - ) - assert "Content-Type: multipart/alternative;" in mail - assert "Content-Type: text/html" in mail - assert "Content-Type: text/plain" in mail - assert devid in mail - assert service_name in mail - assert "${workflow.input." not in mail - assert len(messages) == 4 - - # Sort emails by date - messages.sort(key=lambda x: x["Date"]) - - i = 0 - for service_name in ["crond", "mender-connect"]: - assert_valid_alert( - messages[i], - user_name, - "CRITICAL: Monitor Alert for Service " - + service_name - + " not running on " - + devid, - ) - logger.info( - "test_monitorclient_logs_and_services: got CRITICAL alert email for %s." - % service_name - ) - i = i + 1 - - for service_name in ["crond", "mender-connect"]: - assert_valid_alert( - messages[i], - user_name, - "OK: Monitor Alert for Service " - + service_name - + " not running on " - + devid, - ) - logger.info( - "test_monitorclient_logs_and_services: got OK alert email for %s." - % service_name - ) - i = i + 1 - - assert "${workflow.input" not in mail - logger.info( - "test_monitorclient_logs_and_services: email alert on log file containing a pattern scenario." - ) - log_file = "/tmp/mylog.log" - log_pattern = "session opened for user [a-z]*" - - service_name = "mylog" - prepare_log_monitoring( - mender_device, service_name, log_file, log_pattern, use_ctl=True - ) - prepare_log_monitoring( - mender_device, - service_name + "-tail", - "@tail -f " + log_file, - log_pattern, - use_ctl=True, - ) - mender_device.run("systemctl restart mender-monitor") - time.sleep(2 * wait_for_alert_interval_s) - - mender_device.run("echo 'some line 1' >> " + log_file) - mender_device.run("echo 'some line 2' >> " + log_file) - - time.sleep(2 * wait_for_alert_interval_s) - _, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, 4 - ) - assert 4 == len(messages) - - mender_device.run( - "echo -ne 'a new session opened for user root now\nsome line 3\n' >> " - + log_file - ) - time.sleep(wait_for_alert_interval_s) - - mender_device.run("echo 'some line 4' >> " + log_file) - mender_device.run("echo 'some line 5' >> " + log_file) - - mender_device.run( - "echo -ne 'a new session opened for user root now\nsome line 3\n' >> " - + log_file - ) - time.sleep(wait_for_alert_interval_s) - - time.sleep(2 * wait_for_alert_interval_s) - _, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, 6 - ) - assert len(messages) == 6 - # after MEN-5458 we expect only one critical per log service (mylog, mylog-tail) - for m in messages[-2:]: - assert_valid_alert( - m, - user_name, - "CRITICAL: Monitor Alert for Log file contains " - + log_pattern - + " on " - + devid, - ) - - fds_count_timeout_s = 64 - max_fds_count_diff = 3 - logger.info( - "test_monitorclient_logs_and_services: checking open file descriptors" - ) - time.sleep(fds_count_timeout_s * 0.25) - fds_count0 = mender_device.run( - "ls /proc/$(cat /var/run/monitoring-client.pid)/fd | wc -l" - ) - assert fds_count0.rstrip().isnumeric() - fds_count0 = int(fds_count0.rstrip()) - logger.info( - "test_monitorclient_logs_and_services: currently %s fds open" % fds_count0 - ) - time.sleep(4 * fds_count_timeout_s) - fds_count1 = mender_device.run( - "ls /proc/$(cat /var/run/monitoring-client.pid)/fd | wc -l" - ) - assert fds_count1.rstrip().isnumeric() - fds_count1 = int(fds_count1.rstrip()) - logger.info( - "test_monitorclient_logs_and_services: currently %s fds open" % fds_count0 - ) - assert abs(fds_count1 - fds_count0) <= max_fds_count_diff - logger.info( - "test_monitorclient_logs_and_services: fds count have not increased" - ) - - def test_monitorclient_logs_and_surround(self, monitor_commercial_setup_no_client): - """Tests more lines of logs surrounding a line matching a pattern""" - user_name = "ci.email.tests+{}@mender.io".format(str(uuid.uuid4())) - devid, _, auth, mender_device = self.prepare_env( - monitor_commercial_setup_no_client, user_name - ) - inventory = Inventory(auth) - - logger.info( - "test_monitorclient_logs_and_surround: lines surrounding a pattern scenario." - ) - log_file = "/tmp/mylog.long.log" - log_pattern = "session opened for user \\w+" - service_name = "mylog" - - lines_before_count = 64 - lines_after_count = 64 - for n in range(lines_before_count - 1): - mender_device.run( - "echo 'some long line of logs number: " + str(n) + "' >> " + log_file - ) - mender_device.run( - "echo 'a new session opened for user root now' >> " + log_file - ) - for n in range(lines_after_count - 1): - mender_device.run( - "echo 'some long line of logs number: " + str(n) + "' >> " + log_file - ) - prepare_log_monitoring( - mender_device, - service_name + "-pcre", - log_file, - log_pattern, - use_ctl=True, - ) - time.sleep(2 * wait_for_alert_interval_s) - - alerts, alert_count = self.get_alerts_and_alert_count_for_device( - inventory, devid - ) - assert (True, 1) == (alerts, alert_count) - - _, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, 1 - ) - assert_valid_alert( - messages[0], - user_name, - "CRITICAL: Monitor Alert for Log file contains " - + log_pattern - + " on " - + devid, - ) - device_monitor = DeviceMonitor(auth) - alerts = device_monitor.get_alerts(devid) - assert len(alerts) == 1 - assert len(alerts[0]["subject"]["details"]["lines_before"]) == 30 - assert len(alerts[0]["subject"]["details"]["lines_after"]) == 30 - - def test_monitorclient_logs_and_patterns(self, monitor_commercial_setup_no_client): - """Tests the monitor client email alerting for a Perl compatible regex""" - user_name = "ci.email.tests+{}@mender.io".format(str(uuid.uuid4())) - devid, _, auth, mender_device = self.prepare_env( - monitor_commercial_setup_no_client, user_name - ) - inventory = Inventory(auth) - - logger.info( - "test_monitorclient_logs_and_patterns: email alert on log file containing a pattern scenario." - ) - log_file = "/tmp/mylog.log" - log_pattern = "session opened for user \\w+" - service_name = "mylog" - - mender_device.run("echo 'some line 1' >> " + log_file) - mender_device.run("echo 'some line 2' >> " + log_file) - prepare_log_monitoring( - mender_device, - service_name + "-pcre", - "@tail -f " + log_file, - log_pattern, - use_ctl=True, - ) - mender_device.run("systemctl restart mender-monitor") - time.sleep(2 * wait_for_alert_interval_s) - - mender_device.run( - "echo -ne 'a new session opened for user root now\nsome line 3\n' >> " - + log_file - ) - time.sleep(wait_for_alert_interval_s) - - mender_device.run("echo 'some line 4' >> " + log_file) - mender_device.run("echo 'some line 5' >> " + log_file) - - mender_device.run( - "echo -ne 'a new session opened for user root now\nsome line 5\n' >> " - + log_file - ) - time.sleep(2 * wait_for_alert_interval_s) - - alerts, alert_count = self.get_alerts_and_alert_count_for_device( - inventory, devid - ) - assert (True, 1) == (alerts, alert_count) - - _, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, 1 - ) - assert_valid_alert( - messages[0], - user_name, - "CRITICAL: Monitor Alert for Log file contains " - + log_pattern - + " on " - + devid, - ) - - def test_monitorclient_send_saved_alerts_on_network_issues( - self, monitor_commercial_setup_no_client - ): - """Tests that the client does indeed cache alerts and resend them in the face - of issues, like network connectivity""" - - hostname = os.environ.get("GATEWAY_HOSTNAME", "docker.mender.io") - user_name = "ci.email.tests+{}@mender.io".format(str(uuid.uuid4())) - devid, _, auth, mender_device = self.prepare_env( - monitor_commercial_setup_no_client, user_name - ) - - logger.info( - "test_monitorclient_send_saved_alerts_on_network_issues: email alert on log file containing a pattern scenario with flaky network." - ) - - logger.info( - "test_monitorclient_send_saved_alerts_on_network_issues: disabling access to %s (point to localhost in /etc/hosts)" - % hostname - ) - mender_device.run("sed -i.backup -e '$a127.2.0.1 %s' /etc/hosts" % hostname) - mender_device.run("systemctl restart mender-authd") - - log_file_name = "/tmp/mylog.log" - log_file = "@tail -f " + log_file_name - log_pattern = "session opened for user [a-z]*" - - service_name = "mylog" - prepare_log_monitoring( - mender_device, - service_name, - log_file, - log_pattern, - ) - - # The file needs to exist beforehand, otherwise the monitoring will just exit with a ERRNOEXIST - mender_device.run("touch " + log_file_name) - mender_device.run("systemctl restart mender-monitor") - time.sleep(2 * wait_for_alert_interval_s) - - mender_device.run("echo 'some line 1' >> " + log_file_name) - mender_device.run("echo 'some line 2' >> " + log_file_name) - mender_device.run( - "echo 'a new session opened for user root now' >> " + log_file_name - ) - mender_device.run( - "echo -ne 'some line 4\nsome line 5\nsome line 6\n' >> " + log_file_name - ) - time.sleep(2 * wait_for_alert_interval_s) - mender_device.run( - "echo 'a new session opened for user root now' >> " + log_file_name - ) - mender_device.run("echo -ne 'some line 7\nsomeline 8\n' >> " + log_file_name) - time.sleep(4 * wait_for_alert_interval_s) - - _, messages = get_and_parse_email(monitor_commercial_setup_no_client, user_name) - assert len(messages) == 0 - logger.info( - "test_monitorclient_send_saved_alerts_on_network_issues: got no alerts, device is offline." - ) - - logger.info( - "test_monitorclient_send_saved_alerts_on_network_issues: re-enabling access to %s (restoring /etc/hosts)" - % hostname - ) - mender_device.run("cp /etc/hosts.backup /etc/hosts") - logger.info( - "test_monitorclient_send_saved_alerts_on_network_issues: waiting for alerts to come." - ) - time.sleep(wait_for_alert_interval_s * 10) - - mail, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, 1 - ) - - output = mender_device.run( - "journalctl --unit mender-monitor --output cat --no-pager --reverse" - ) - - assert len(messages) == 1, output - for m in [messages[0]]: - assert_valid_alert( - m, - user_name, - "CRITICAL: Monitor Alert for Log file contains " - + log_pattern - + " on " - + devid, - ) - logger.info( - "test_monitorclient_send_saved_alerts_on_network_issues: got CRITICAL alert email." - ) - - assert_valid_alert( - messages[-1], - user_name, - "CRITICAL: Monitor Alert for Log file contains " - + log_pattern - + " on " - + devid, - ) - assert not "${workflow.input" in mail - logger.info( - "test_monitorclient_send_saved_alerts_on_network_issues: got CRITICAL alert email." - ) - - def test_monitorclient_send_configuration_data( - self, monitor_commercial_setup_no_client - ): - """Tests the monitor client configuration push""" - user_name = "ci.email.tests+{}@mender.io".format(str(uuid.uuid4())) - devid, _, auth, mender_device = self.prepare_env( - monitor_commercial_setup_no_client, user_name - ) - - logger.info( - "test_monitorclient_send_configuration_data: push configuration scenario." - ) - mender_device.run( - "sed -i.backup -e 's/CONFPUSH_INTERVAL=.*/CONFPUSH_INTERVAL=8/' /usr/share/mender-monitor/config/config.sh" - ) - prepare_service_monitoring(mender_device, "crond", use_ctl=True) - prepare_service_monitoring(mender_device, "dbus", use_ctl=True) - prepare_log_monitoring( - mender_device, "syslog", "/var/log/syslog", "root.*access", use_ctl=True - ) - prepare_log_monitoring( - mender_device, - "clientlogs", - "@journalctl -u mender-authd -f", - "[Ee]rror.*", - use_ctl=True, - ) - mender_device.run("systemctl restart mender-monitor") - device_monitor = DeviceMonitor(auth) - wait_iterations = wait_for_alert_interval_s + 10 - while wait_iterations > 0: - time.sleep(1) - configuration = device_monitor.get_configuration(devid) - if len(configuration) == 4: - break - wait_iterations = wait_iterations - 1 - - assert len(configuration) == 4 - for entity in [ - {"name": "crond.sh", "type": "service"}, - {"name": "dbus.sh", "type": "service"}, - {"name": "syslog.sh", "type": "log"}, - {"name": "clientlogs.sh", "type": "log"}, - ]: - found = False - for c in configuration: - assert "name" in c - assert "type" in c - assert "status" in c - assert c["status"] == "enabled" - if entity["name"] == c["name"] and entity["type"] == c["type"]: - found = True - break - assert found - - def test_monitorclient_remove_old_alerts(self, monitor_commercial_setup_no_client): - """Tests the removal of older alerts from the persistent store""" - alert_resend_interval_s = 4 - alert_max_age = 16 - user_name = "ci.email.tests+{}@mender.io".format(str(uuid.uuid4())) - _, _, _, mender_device = self.prepare_env( - monitor_commercial_setup_no_client, user_name - ) - - logger.info( - "test_monitorclient_remove_old_alerts: remove old alerts from store scenario." - ) - - # Stop mender-auth so that mender-monitor cannot send alerts - # Stop also mender-updated to prevent dbus-daemon to automatically start mender-authd - mender_device.run("systemctl stop mender-authd mender-updated") - - mender_device.run( - "sed -i.backup -e 's/ALERT_STORE_MAX_RECORD_AGE_S=.*/ALERT_STORE_MAX_RECORD_AGE_S=" - + str(alert_max_age) - + "/' " - + "-e 's/DEFAULT_ALERT_STORE_RESEND_INTERVAL_S=.*/DEFAULT_ALERT_STORE_RESEND_INTERVAL_S=" - + str(alert_resend_interval_s) - + "/' " - + "-e 's/SEND_ALERT_MAX_INTERVAL_S=.*/SEND_ALERT_MAX_INTERVAL_S=0" - + "/' /usr/share/mender-monitor/config/config.sh" - ) - - # Insert 4 alerts: 2 and 2 separated 8s. - # Then check the expiration with good tolerance margins. - # Stop and start mender-monitor to remove [part of] the restart latency - # - # 0 4 8 12 16 20 24 28 - # |---|---|---|---|---|---|---|---| - # ^ ^ ^ ^ - # |-------|------> | - # |--------------> - # - # T0: insert alerts key1, key2 - # T8: insert alerts key3, key4 - # T16: key1, key2 expired - # T24: key3, key4 expired - mender_device.run("systemctl stop mender-monitor") - mender_device.run( - "bash -c 'cd /usr/share/mender-monitor && . lib/fixlenstore-lib.sh;" - + "fixlenstore_put key1; fixlenstore_put key2;" - + "sleep 8; " - + "fixlenstore_put key3; fixlenstore_put key4;'" - ) - mender_device.run("systemctl start mender-monitor") - - # T8: mender-monitor started - time.sleep(alert_resend_interval_s) - time.sleep(alert_resend_interval_s) - - # Shift by 1s to avoid race condition when checking - time.sleep(1) - - # T16+1: key1, key2 expired - output = mender_device.run( - "bash -c 'cd /usr/share/mender-monitor && . lib/fixlenstore-lib.sh;" - + "keys_nolock | wc -l;'" - ) - logger.info("test_monitorclient_remove_old_alerts got %s keys" % output) - assert output == "2\n" - - time.sleep(alert_resend_interval_s) - time.sleep(alert_resend_interval_s) - # T24+1: key3, key4 expired - output = mender_device.run( - "bash -c 'cd /usr/share/mender-monitor && . lib/fixlenstore-lib.sh;" - + "keys_nolock | wc -l;'" - ) - logger.info("test_monitorclient_remove_old_alerts got %s keys" % output) - assert output == "0\n" - - mender_device.run( - "mv /usr/share/mender-monitor/config/config.sh.backup /usr/share/mender-monitor/config/config.sh" - ) - mender_device.run("systemctl restart mender-monitor") - - def test_monitorclient_alert_store_discard_http_400( - self, monitor_commercial_setup_no_client - ): - """Tests that malformed alerts in the store (HTTP 400) are discarded""" - service_name = "crond" - hostname = os.environ.get("GATEWAY_HOSTNAME", "docker.mender.io") - user_name = "ci.email.tests+{}@mender.io".format(str(uuid.uuid4())) - devid, _, _, mender_device = self.prepare_env( - monitor_commercial_setup_no_client, user_name - ) - - prepare_service_monitoring(mender_device, service_name) - time.sleep(2 * wait_for_alert_interval_s) - - logger.info( - "test_monitorclient_alert_store_discard_http_400: disabling access to %s (point to localhost in /etc/hosts)" - % hostname - ) - mender_device.run("sed -i.backup -e '$a127.2.0.1 %s' /etc/hosts" % hostname) - mender_device.run("systemctl restart mender-authd") - mender_device.run("systemctl stop %s" % service_name) - logger.info( - "Stopped %s, sleeping %ds." % (service_name, wait_for_alert_interval_s) - ) - time.sleep(wait_for_alert_interval_s) - mender_device.run("systemctl start %s" % service_name) - logger.info( - "Started %s, sleeping %ds" % (service_name, wait_for_alert_interval_s) - ) - - time.sleep(2 * wait_for_alert_interval_s) - - _, messages = get_and_parse_email(monitor_commercial_setup_no_client, user_name) - assert len(messages) == 0 - logger.info( - "test_monitorclient_alert_store_discard_http_400: got no alerts, device is offline." - ) - - logger.info( - "test_monitorclient_alert_store_discard_http_400: manipulating local store." - ) - num_alerts_valid = mender_device.run( - "bash -c '" - "cd /usr/share/mender-monitor;" - ". lib/fixlenstore-lib.sh;" - "fixlenstore_count;" - "'" - ).strip() - assert int(num_alerts_valid) == 2 - num_alerts_corrupted = mender_device.run( - "bash -c '" - "cd /usr/share/mender-monitor;" - ". lib/fixlenstore-lib.sh;" - 'fixlenstore_put "invalid json break here";' - 'fixlenstore_put "something else there";' - "fixlenstore_count;" - "'" - ).strip() - assert int(num_alerts_corrupted) == 4 - - logger.info( - "test_monitorclient_alert_store_discard_http_400: re-enabling access to %s (restoring /etc/hosts)" - % hostname - ) - mender_device.run("mv /etc/hosts.backup /etc/hosts") - logger.info( - "test_monitorclient_alert_store_discard_http_400: waiting for alerts (not) to come." - ) - time.sleep(8 * wait_for_alert_interval_s) - - _, messages = get_and_parse_email(monitor_commercial_setup_no_client, user_name) - assert len(messages) == 0 - logger.info( - "test_monitorclient_alert_store_discard_http_400: got no alerts, were they discarded?" - ) - - num_alerts_after_discard = mender_device.run( - "bash -c '" - "cd /usr/share/mender-monitor;" - ". lib/fixlenstore-lib.sh;" - "fixlenstore_count;" - "'" - ).strip() - assert int(num_alerts_after_discard) == 0 - logger.info( - "test_monitorclient_alert_store_discard_http_400: store is empty, they were discarded!" - ) - - logger.info( - "test_monitorclient_alert_store_discard_http_400: disabling again access to %s (point to localhost in /etc/hosts)" - % hostname - ) - mender_device.run("sed -i.backup -e '$a127.2.0.1 %s' /etc/hosts" % hostname) - mender_device.run("systemctl restart mender-authd") - mender_device.run("systemctl stop %s" % service_name) - logger.info( - "Stopped %s, sleeping %ds." % (service_name, wait_for_alert_interval_s) - ) - expected_alerts_count = 1 # one for CRITICAL, because we stopped the service - time.sleep(wait_for_alert_interval_s) - mender_device.run("systemctl start %s" % service_name) - logger.info( - "Started %s, sleeping %ds" % (service_name, wait_for_alert_interval_s) - ) - expected_alerts_count = ( - expected_alerts_count + 1 - ) # one for OK, because we started the service - time.sleep(2 * wait_for_alert_interval_s) - - _, messages = get_and_parse_email(monitor_commercial_setup_no_client, user_name) - assert len(messages) == 0 - logger.info( - "test_monitorclient_alert_store_discard_http_400: got no alerts, device is offline." - ) - - logger.info( - "test_monitorclient_alert_store_discard_http_400: re-enabling again access to %s (restoring /etc/hosts)" - % hostname - ) - mender_device.run("mv /etc/hosts.backup /etc/hosts") - logger.info( - "test_monitorclient_alert_store_discard_http_400: waiting for alerts to come." - ) - time.sleep(8 * wait_for_alert_interval_s) - - mail, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, - user_name, - expected_alerts_count, - ) - assert len(messages) >= expected_alerts_count - - # Sort emails by date - messages.sort(key=lambda x: x["Date"]) - - assert "${workflow.input" not in mail - assert_valid_alert( - messages[0], - user_name, - "CRITICAL: Monitor Alert for Service " - + service_name - + " not running on " - + devid, - ) - logger.info( - "test_monitorclient_alert_store_discard_http_400: got CRITICAL alert email." - ) - - assert_valid_alert( - messages[1], - user_name, - "OK: Monitor Alert for Service " - + service_name - + " not running on " - + devid, - ) - logger.info( - "test_monitorclient_alert_store_discard_http_400: got OK alert email." - ) - - def mock_docker_events(self, mender_device): - docker_events_exec = """ -#!/bin/bash - -while [ ! -f /tmp/docker_restart ]; do - sleep 1; -done - -cat < 0 - assert_valid_alert( - messages[0], - user_name, - "CRITICAL: Monitor Alert for Docker container " - + container_name - + " " - + action - + " on " - + devid, - ) - assert "${workflow.input." not in mail - logger.info("test_monitorclient_dockerevents: got CRITICAL alert email.") - - mender_device.run("rm -f /tmp/docker_restart") - logger.info( - "test_monitorclient_dockerevents: emulated restarts finished. waiting for OK." - ) - time.sleep(alert_expiration_time_seconds) - - alerts, alert_count = self.get_alerts_and_alert_count_for_device( - inventory, devid - ) - assert (False, 0) == (alerts, alert_count) - - mail, messages = get_and_parse_email_n( - monitor_commercial_setup_no_client, user_name, 1 - ) - messages_count = len(messages) - assert messages_count > 0 - assert_valid_alert( - messages[1], - user_name, - "OK: Monitor Alert for Docker container " - + container_name - + " " - + action - + " on " - + devid, - ) - assert "${workflow.input" not in mail - logger.info("test_monitorclient_dockerevents: got OK alert email.") diff --git a/tests/tests/test_mtls.py b/tests/tests/test_mtls.py deleted file mode 100644 index e833636d7..000000000 --- a/tests/tests/test_mtls.py +++ /dev/null @@ -1,418 +0,0 @@ -# Copyright 2023 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import os.path -import pytest -import re -import shutil -import tempfile -import time - -from testutils.common import create_org -from testutils.infra.container_manager import factory -from testutils.infra.device import MenderDevice - -from .. import conftest -from ..MenderAPI import reset_mender_api, auth, deploy, devauth, logger -from ..helpers import Helpers -from .common_artifact import get_script_artifact -from .mendertesting import MenderTesting - -container_factory = factory.get_factory() - - -@pytest.fixture(scope="function") -def setup_ent_mtls(request): - env = container_factory.get_mtls_setup() - request.addfinalizer(env.teardown) - env.setup() - - mtls_username = "mtls@mender.io" - mtls_password = "correcthorsebatterystaple" - - env.tenant = create_org( - "Mender", - mtls_username, - mtls_password, - containers_namespace=env.name, - container_manager=env, - ) - env.user = env.tenant.users[0] - env.start_mtls_gateway() - - reset_mender_api(env) - - auth.username = mtls_username - auth.password = mtls_password - auth.multitenancy = True - auth.current_tenant = env.tenant - - env.stop_api_gateway() - - # start a new mender client - env.new_mtls_client("mender-client", env.tenant.tenant_token) - env.device = MenderDevice(env.get_mender_clients()[0]) - env.device.ssh_is_opened() - - return env - - -def make_script_artifact(artifact_name, device_type, output_path): - script = b"""\ -#! /bin/bash - -set -xe - -# Just give it a little bit of time -sleep 6s - -# Successful update -exit 0 -""" - return get_script_artifact(script, artifact_name, device_type, output_path) - - -class TestClientMTLSEnterprise: - wait_for_device_timeout_seconds = 64 - - def hsm_setup(self, pin, ssl_engine_id, device): - algorithm = "rsa" - key = f"/var/lib/mender/client.1.{algorithm}.key" - script = f"""\ -#!/bin/bash -softhsm2-util --init-token --free --label unittoken1 --pin {pin} --so-pin 0002 -pkcs11-tool --module /usr/lib/softhsm/libsofthsm2.so --login --pin {pin} --write-object "{key}" --type privkey --id 0909 --label privatekey -""" - tmpdir = tempfile.mkdtemp() - initialize_hsm_script = os.path.join(tmpdir, "init-hsm.sh") - try: - with open(initialize_hsm_script, "w") as fd: - fd.write(script) - device.put( - os.path.basename(initialize_hsm_script), - local_path=os.path.dirname(initialize_hsm_script), - remote_path="/tmp", - ) - device.run("chmod 755 /tmp/" + os.path.basename(initialize_hsm_script)) - device.run("/tmp/" + os.path.basename(initialize_hsm_script)) - finally: - shutil.rmtree(tmpdir) - - def setup_openssl_conf(self, device, hsm_implementation): - device.run("cp /etc/ssl/openssl.cnf /etc/ssl/openssl.cnf.backup") - if hsm_implementation == "engine": - conf = """ -[openssl_init] -engines = engine_section - -[engine_section] -pkcs11 = pkcs11_section - -[pkcs11_section] -engine_id = pkcs11 -MODULE_PATH = /usr/lib/softhsm/libsofthsm2.so - """ - - device.run(f'echo -ne "{conf}" >> /etc/ssl/openssl.cnf') - elif hsm_implementation == "provider": - conf = """ -[openssl_init] -providers = provider_sect - -[provider_sect] -default = default_sect -pkcs11 = pkcs11_sect - -[pkcs11_sect] -module = /usr/lib/ossl-modules/pkcs11.so -pkcs11-module-path = /usr/lib/softhsm/libsofthsm2.so -activate = 1 - -[default_sect] -activate = 1 - """ - device.run(f'echo -ne "{conf}" >> /etc/ssl/openssl.cnf') - - def hsm_get_key_uri(self, pin, ssl_engine_id, device): - pt11tool_output = device.run( - "p11tool --login --provider=/usr/lib/softhsm/libsofthsm2.so --set-pin=" - + pin - + " --list-all-privkeys" - ).rstrip("\n") - key_uri = re.search(r"URL:\s(.*)", pt11tool_output).group(1) - key_uri = key_uri + ";pin-value=" + pin - - return key_uri - - def hsm_cleanup(self, device): - device.run("mv /etc/ssl/openssl.cnf.backup /etc/ssl/openssl.cnf || true") - - def common_test_mtls_enterprise( - self, env, algorithm=None, use_hsm=False, use_engine=True - ): - # upload the certificates - basedir = os.path.join( - os.path.dirname(__file__), - "..", - "..", - ) - certs = os.path.join( - basedir, - "extra", - "mtls", - "certs", - ) - - env.device.put( - "tenant.ca.crt", - local_path=os.path.join(certs, "tenant-ca"), - remote_path="/etc/ssl/certs", - ) - env.device.run("update-ca-certificates") - - if algorithm is not None: - env.device.put( - f"client.1.{algorithm}.crt", - local_path=os.path.join(certs, "client"), - remote_path="/var/lib/mender", - ) - env.device.put( - f"client.1.{algorithm}.key", - local_path=os.path.join(certs, "client"), - remote_path="/var/lib/mender", - ) - - # Stop also mender-updated to prevent dbus-daemon to automatically start mender-authd - env.device.run("systemctl stop mender-authd mender-updated") - tmpdir = tempfile.mkdtemp() - - ssl_engine_id = "" - if use_engine: - ssl_engine_id = "pkcs11" - pin = "0001" - if algorithm is not None and use_hsm is True: - self.hsm_setup(pin, ssl_engine_id, env.device) - key_uri = self.hsm_get_key_uri(pin, ssl_engine_id, env.device) - - # Install the script update module required for this test - Helpers.install_community_update_module(env.device, "script") - - try: - # retrieve the original configuration file - output = env.device.run("cat /etc/mender/mender.conf") - config = json.loads(output) - # replace mender.conf with an mTLS enabled one - config["ServerURL"] = "https://mtls-gateway:8080" - config["SkipVerify"] = True - if algorithm is not None: - if use_hsm is True: - config["HttpsClient"] = { - "SSLEngine": ssl_engine_id, - "Certificate": f"/var/lib/mender/client.1.{algorithm}.crt", - "Key": key_uri, - } - config["Security"] = { - "SSLEngine": ssl_engine_id, - "AuthPrivateKey": key_uri, - } - logger.info('client key set to "%s"' % key_uri) - else: - config["Security"] = { - "AuthPrivateKey": f"/var/lib/mender/client.1.{algorithm}.key", - } - config["HttpsClient"] = { - "Certificate": f"/var/lib/mender/client.1.{algorithm}.crt", - "Key": f"/var/lib/mender/client.1.{algorithm}.key", - } - if "ArtifactVerifyKey" in config: - del config["ArtifactVerifyKey"] - mender_conf = os.path.join(tmpdir, "mender.conf") - with open(mender_conf, "w") as fd: - json.dump(config, fd) - env.device.put( - os.path.basename(mender_conf), - local_path=os.path.dirname(mender_conf), - remote_path="/etc/mender", - ) - finally: - shutil.rmtree(tmpdir) - - # start the api gateway - env.start_api_gateway() - - # start the Mender client - logger.info("starting the client.") - env.device.run("systemctl daemon-reload") - env.device.run("systemctl start mender-authd mender-updated") - - @MenderTesting.fast - @pytest.mark.parametrize("algorithm", ["rsa", "ec256", "ed25519"]) - def test_mtls_enterprise(self, setup_ent_mtls, algorithm): - - self.common_test_mtls_enterprise(setup_ent_mtls, algorithm, use_hsm=False) - - # prepare a test artifact - with tempfile.NamedTemporaryFile() as tf: - artifact = make_script_artifact( - "mtls-artifact", conftest.machine_name, tf.name - ) - deploy.upload_image(artifact) - - for device in devauth.get_devices_status("pending"): - devauth.decommission(device["id"]) - - i = self.wait_for_device_timeout_seconds - while i > 0: - i = i - 1 - time.sleep(1) - devices = list( - set([device["id"] for device in devauth.get_devices_status("accepted")]) - ) - if len(devices) > 0: - break - - # deploy the update to the device - devices = list( - set([device["id"] for device in devauth.get_devices_status("accepted")]) - ) - assert len(devices) == 1 - deployment_id = deploy.trigger_deployment( - "mtls-test", - artifact_name="mtls-artifact", - devices=devices, - ) - - # now just wait for the update to succeed - deploy.check_expected_statistics(deployment_id, "success", 1) - deploy.check_expected_status("finished", deployment_id) - - # verify the update was actually installed on the device - out = setup_ent_mtls.device.run("mender-update show-artifact").strip() - assert out == "mtls-artifact" - - @MenderTesting.fast - @pytest.mark.parametrize("algorithm", ["rsa"]) - @pytest.mark.parametrize( - "hsm_implementation", - ["engine", "provider"], - ) - def test_mtls_enterprise_hsm(self, setup_ent_mtls, algorithm, hsm_implementation): - # Check if the device has SoftHSM (from yocto dunfell forward) - output = setup_ent_mtls.device.run( - "test -e /usr/lib/softhsm/libsofthsm2.so && echo true", hide=True - ) - if output.rstrip() != "true": - pytest.fail("Needs SoftHSM to run this test") - - # Check if the device has PKCS#11 module to interface as provider - # yocto kirkstone and older cannot run this test. Skip instead of fail. - output = setup_ent_mtls.device.run( - "test -e /usr/lib/ossl-modules/pkcs11.so && echo true", hide=True - ) - if output.rstrip() != "true" and hsm_implementation == "provider": - pytest.skip("Needs PKCS#11 module. Skipping") - - self.setup_openssl_conf(setup_ent_mtls.device, hsm_implementation) - - try: - self.common_test_mtls_enterprise( - setup_ent_mtls, - algorithm, - use_hsm=True, - use_engine=("engine" == hsm_implementation), - ) - - # prepare a test artifact - with tempfile.NamedTemporaryFile() as tf: - artifact = make_script_artifact( - "mtls-artifact", conftest.machine_name, tf.name - ) - - # prepare a test artifact - with tempfile.NamedTemporaryFile() as tf: - artifact = make_script_artifact( - "mtls-artifact", conftest.machine_name, tf.name - ) - deploy.upload_image(artifact) - - for device in devauth.get_devices_status("pending"): - devauth.decommission(device["id"]) - - i = self.wait_for_device_timeout_seconds - while i > 0: - i = i - 1 - time.sleep(1) - devices = list( - set( - [ - device["id"] - for device in devauth.get_devices_status("accepted") - ] - ) - ) - if len(devices) > 0: - break - - # deploy the update to the device - devices = list( - set( - [ - device["id"] - for device in devauth.get_devices_status("accepted") - ] - ) - ) - assert len(devices) == 1 - deployment_id = deploy.trigger_deployment( - "mtls-test", - artifact_name="mtls-artifact", - devices=devices, - ) - - # now just wait for the update to succeed - deploy.check_expected_statistics(deployment_id, "success", 1) - deploy.check_expected_status("finished", deployment_id) - - # verify the update was actually installed on the device - out = setup_ent_mtls.device.run("mender-update show-artifact").strip() - assert out == "mtls-artifact" - finally: - self.hsm_cleanup(setup_ent_mtls.device) - - @MenderTesting.fast - def test_mtls_enterprise_without_client_cert(self, setup_ent_mtls): - # set up the mTLS test environment, without providing client certs - self.common_test_mtls_enterprise(setup_ent_mtls, algorithm=None, use_hsm=False) - - # in here it also may happen, that the client is started earlier, and device registers - # as pending. in that case get_devices_status which is called from get_devices will - # loop until it runs out of iterations, due to the fact that we expect to have 0 devices. - # to prevent this from happening, lets wait a bit, if the device shows as pending, - # if it does, decommission it, and then restart the client, and wait to be sure the - # device will not re-appear, which is the main idea of the test. - device_not_present_timeout_seconds = 30 - for device in devauth.get_devices_status( - "pending", max_wait=device_not_present_timeout_seconds * 0.5, no_assert=True - ): - devauth.decommission(device["id"]) - - setup_ent_mtls.device.run("systemctl start mender-authd") - - # wait device_not_present_timeout_seconds - time.sleep(device_not_present_timeout_seconds) - - # no device shows up, because mTLS doesn't forward requests to the backend - devauth.get_devices(expected_devices=0) diff --git a/tests/tests/test_os_ent_migration.py b/tests/tests/test_os_ent_migration.py deleted file mode 100644 index 82959e85a..000000000 --- a/tests/tests/test_os_ent_migration.py +++ /dev/null @@ -1,212 +0,0 @@ -# Copyright 2022 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import time -import json - -import pytest - -from testutils.common import create_user, make_accepted_device, User, Tenant -from testutils.api.client import ApiClient -from testutils.infra.cli import CliTenantadm -import testutils.api.deviceauth as deviceauth -import testutils.api.deployments as deployments -import testutils.api.useradm as useradm -from testutils.infra.container_manager import factory - -# This test requires special manipulation of containers, so it will use -# directly the factory to prepare fixtures instead of common_setup -container_factory = factory.get_factory() - - -@pytest.fixture(scope="class") -def initial_os_setup(request): - """Start the minimum OS setup, create some uses and devices. - Return {"os_devs": [...], "os_users": [...]} - """ - os_env = container_factory.get_standard_setup(num_clients=0) - # We will later re-create other environments, but this one (or any, really) will be - # enough for the teardown if we keep using the same namespace. - request.addfinalizer(os_env.teardown) - - os_env.setup() - os_env.init_data = initialize_os_setup(os_env) - - return os_env - - -@pytest.fixture(scope="class") -def initial_enterprise_setup(initial_os_setup): - """ - Start ENT for the first time (no tenant yet). - """ - initial_os_setup.teardown_exclude(["mender-mongo"]) - - # Create a new env reusing the same namespace - ent_no_tenant_env = container_factory.get_enterprise_setup( - initial_os_setup.name, num_clients=0 - ) - ent_no_tenant_env.setup() - - return initial_os_setup - - -@pytest.fixture(scope="class") -def migrated_enterprise_setup(initial_enterprise_setup): - """ - Create an org (tenant + user), restart with default tenant token. - The ENT setup is ready for tests. - Return {"os_devs": [...], "os_users": [...], "tenant": } - """ - ent_data = migrate_ent_setup(initial_enterprise_setup) - - # preserve the user/tenant created before restart - initial_enterprise_setup.teardown_exclude(["mender-mongo"]) - - # Create a new env reusing the same namespace - ent_with_tenant_env = container_factory.get_enterprise_setup( - initial_enterprise_setup.name, num_clients=0 - ) - ent_with_tenant_env.setup( - recreate=False, - env={"DEFAULT_TENANT_TOKEN": "%s" % ent_data["tenant"].tenant_token}, - ) - - initial_enterprise_setup.init_data = dict( - {**ent_data, **initial_enterprise_setup.init_data} - ) - return initial_enterprise_setup - - -def initialize_os_setup(env): - """Seed the OS setup with all operational data - users and devices. - Return {"os_devs": [...], "os_users": [...]} - """ - uadmm = ApiClient(useradm.URL_MGMT, host=env.get_mender_gateway()) - dauthd = ApiClient(deviceauth.URL_DEVICES, host=env.get_mender_gateway()) - dauthm = ApiClient(deviceauth.URL_MGMT, host=env.get_mender_gateway()) - - users = [ - create_user("foo@tenant.com", "correcthorse", containers_namespace=env.name), - create_user("bar@tenant.com", "correcthorse", containers_namespace=env.name), - ] - - r = uadmm.call("POST", useradm.URL_LOGIN, auth=(users[0].name, users[0].pwd)) - - assert r.status_code == 200 - utoken = r.text - - # create and accept some devs; save tokens - devs = [] - for _ in range(10): - devs.append(make_accepted_device(dauthd, dauthm, utoken)) - - # get tokens for all - for d in devs: - body, sighdr = deviceauth.auth_req( - d.id_data, d.authsets[0].pubkey, d.authsets[0].privkey - ) - - r = dauthd.call("POST", deviceauth.URL_AUTH_REQS, body, headers=sighdr) - - assert r.status_code == 200 - d.token = r.text - - return {"os_devs": devs, "os_users": users} - - -def migrate_ent_setup(env): - """Migrate the ENT setup - create a tenant and user via create-org, - substitute default token env in the ent. testing layer. - """ - # extra long sleep to make sure all services ran their migrations - # maybe conductor fails because some services are still in a migration phase, - # and not serving the API yet? - time.sleep(30) - - u = User("", "baz@tenant.com", "correcthorse") - - cli = CliTenantadm(containers_namespace=env.name) - tid = cli.create_org("tenant", u.name, u.pwd) - time.sleep(10) - - tenant = cli.get_tenant(tid) - - tenant = json.loads(tenant) - ttoken = tenant["tenant_token"] - - t = Tenant("tenant", tid, ttoken) - t.users.append(u) - - return {"tenant": t} - - -@pytest.mark.usefixtures("migrated_enterprise_setup") -class TestEnterpriseMigration: - def test_users_ok(self, migrated_enterprise_setup): - mender_gateway = migrated_enterprise_setup.get_mender_gateway() - uadmm = ApiClient(useradm.URL_MGMT, host=mender_gateway) - - # os users can't log in - for u in migrated_enterprise_setup.init_data["os_users"]: - r = uadmm.call("POST", useradm.URL_LOGIN, auth=(u.name, u.pwd)) - assert r.status_code == 401 - - # but enterprise user can - ent_user = migrated_enterprise_setup.init_data["tenant"].users[0] - r = uadmm.call("POST", useradm.URL_LOGIN, auth=(ent_user.name, ent_user.pwd)) - assert r.status_code == 200 - - def test_devs_ok(self, migrated_enterprise_setup): - mender_gateway = migrated_enterprise_setup.get_mender_gateway() - uadmm = ApiClient(useradm.URL_MGMT, host=mender_gateway) - dauthd = ApiClient(deviceauth.URL_DEVICES, host=mender_gateway) - dauthm = ApiClient(deviceauth.URL_MGMT, host=mender_gateway) - depld = ApiClient(deployments.URL_DEVICES, host=mender_gateway) - - # current dev tokens don't work right off the bat - # the deviceauth db is empty - for d in migrated_enterprise_setup.init_data["os_devs"]: - resp = depld.with_auth(d.token).call( - "GET", - deployments.URL_NEXT, - qs_params={"artifact_name": "foo", "device_type": "bar"}, - ) - - assert resp.status_code == 401 - - # but even despite the 'dummy' tenant token - # os devices can get into the deviceauth db for acceptance - ent_user = migrated_enterprise_setup.init_data["tenant"].users[0] - r = uadmm.call("POST", useradm.URL_LOGIN, auth=(ent_user.name, ent_user.pwd)) - assert r.status_code == 200 - utoken = r.text - - for d in migrated_enterprise_setup.init_data["os_devs"]: - body, sighdr = deviceauth.auth_req( - d.id_data, - d.authsets[0].pubkey, - d.authsets[0].privkey, - tenant_token="dummy", - ) - - r = dauthd.call("POST", deviceauth.URL_AUTH_REQS, body, headers=sighdr) - - assert r.status_code == 401 - - r = dauthm.with_auth(utoken).call( - "GET", deviceauth.URL_MGMT_DEVICES, path_params={"id": d.id} - ) - - assert r.status_code == 200 - assert len(r.json()) == len(migrated_enterprise_setup.init_data["os_devs"]) diff --git a/tests/tests/test_preauth.py b/tests/tests/test_preauth.py deleted file mode 100644 index 5c2813116..000000000 --- a/tests/tests/test_preauth.py +++ /dev/null @@ -1,249 +0,0 @@ -# Copyright 2021 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import time -import uuid -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import serialization - -from ..common_setup import standard_setup_one_client, enterprise_no_client -from .mendertesting import MenderTesting -from ..MenderAPI import auth, devauth, inv, logger -from ..helpers import Helpers -from testutils.infra.device import MenderDevice - - -class TestPreauthBase(MenderTesting): - def do_test_ok_preauth_and_bootstrap(self, container_manager): - """ - Test the happy path from preauthorizing a device to a successful bootstrap. - Verify that the device/auth set appear correctly in devauth API results. - """ - mender_device = container_manager.device - - # we'll use the same pub key for the preauth'd device, so get it - preauth_key = Client.get_pub_key(mender_device) - - # preauthorize a new device - preauth_iddata = {"mac": "mac-preauth"} - # serialize manually to avoid an extra space (id data helper doesn't insert one) - preauth_iddata_str = '{"mac":"mac-preauth"}' - - r = devauth.preauth(json.loads(preauth_iddata_str), preauth_key) - assert r.status_code == 201 - - # verify the device appears correctly in api results - devs = devauth.get_devices(2) - - dev_preauth = [d for d in devs if d["status"] == "preauthorized"] - assert len(dev_preauth) == 1 - dev_preauth = dev_preauth[0] - logger.info("dev_prauth_map: " + str(dev_preauth)) - assert dev_preauth["identity_data"] == preauth_iddata - assert len(dev_preauth["auth_sets"]) == 1 - assert dev_preauth["auth_sets"][0]["pubkey"] == preauth_key - - # make one of the existing devices the preauthorized device - # by substituting id data script - Client.substitute_id_data(mender_device, preauth_iddata) - - # verify api results - after some time the device should be 'accepted' - for _ in range(120): - time.sleep(15) - dev_accepted = devauth.get_devices_status( - status="accepted", expected_devices=2 - ) - if len([d for d in dev_accepted if d["status"] == "accepted"]) == 1: - break - - logger.info("devices: " + str(dev_accepted)) - dev_accepted = [d for d in dev_accepted if d["status"] == "accepted"] - logger.info("accepted devices: " + str(dev_accepted)) - - Client.get_logs(mender_device) - - assert len(dev_accepted) == 1, "looks like the device was never accepted" - dev_accepted = dev_accepted[0] - logger.info("accepted device: " + str(dev_accepted)) - - assert dev_accepted["identity_data"] == preauth_iddata - assert len(dev_preauth["auth_sets"]) == 1 - assert dev_accepted["auth_sets"][0]["pubkey"] == preauth_key - - # verify device was issued a token - Helpers.check_log_is_authenticated(mender_device) - - def do_test_ok_preauth_and_remove(self): - """ - Test the removal of a preauthorized auth set, verify it's gone from all API results. - """ - # preauthorize - preauth_iddata = json.loads('{"mac":"preauth-mac"}') - preauth_key = """-----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzogVU7RGDilbsoUt/DdH -VJvcepl0A5+xzGQ50cq1VE/Dyyy8Zp0jzRXCnnu9nu395mAFSZGotZVr+sWEpO3c -yC3VmXdBZmXmQdZqbdD/GuixJOYfqta2ytbIUPRXFN7/I7sgzxnXWBYXYmObYvdP -okP0mQanY+WKxp7Q16pt1RoqoAd0kmV39g13rFl35muSHbSBoAW3GBF3gO+mF5Ty -1ddp/XcgLOsmvNNjY+2HOD5F/RX0fs07mWnbD7x+xz7KEKjF+H7ZpkqCwmwCXaf0 -iyYyh1852rti3Afw4mDxuVSD7sd9ggvYMc0QHIpQNkD4YWOhNiE1AB0zH57VbUYG -UwIDAQAB ------END PUBLIC KEY----- -""" - - r = devauth.preauth(preauth_iddata, preauth_key) - assert r.status_code == 201 - - devs = devauth.get_devices(2) - - dev_preauth = [d for d in devs if d["identity_data"] == preauth_iddata] - assert len(dev_preauth) == 1 - dev_preauth = dev_preauth[0] - - # remove from deviceauth - r = devauth.delete_auth_set( - dev_preauth["id"], dev_preauth["auth_sets"][0]["id"] - ) - assert r.status_code == 204 - - # verify removed from deviceauth - devs = devauth.get_devices(1) - dev_removed = [d for d in devs if d["identity_data"] == preauth_iddata] - assert len(dev_removed) == 0 - - # verify removed from deviceauth - r = devauth.get_device(dev_preauth["id"]) - assert r.status_code == 404 - - # verify removed from inventory - r = inv.get_device(dev_preauth["id"]) - assert r.status_code == 404 - - def do_test_fail_preauth_existing(self): - """ - Test 'conflict' response when an identity data set already exists. - """ - # wait for the device to appear - devs = devauth.get_devices(1) - dev = devs[0] - - # try to preauthorize the same id data, new key - preauth_key = """-----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzogVU7RGDilbsoUt/DdH -VJvcepl0A5+xzGQ50cq1VE/Dyyy8Zp0jzRXCnnu9nu395mAFSZGotZVr+sWEpO3c -yC3VmXdBZmXmQdZqbdD/GuixJOYfqta2ytbIUPRXFN7/I7sgzxnXWBYXYmObYvdP -okP0mQanY+WKxp7Q16pt1RoqoAd0kmV39g13rFl35muSHbSBoAW3GBF3gO+mF5Ty -1ddp/XcgLOsmvNNjY+2HOD5F/RX0fs07mWnbD7x+xz7KEKjF+H7ZpkqCwmwCXaf0 -iyYyh1852rti3Afw4mDxuVSD7sd9ggvYMc0QHIpQNkD4YWOhNiE1AB0zH57VbUYG -UwIDAQAB ------END PUBLIC KEY----- -""" - r = devauth.preauth(dev["identity_data"], preauth_key) - assert r.status_code == 409 - - -class TestPreauth(TestPreauthBase): - def test_ok_preauth_and_bootstrap(self, standard_setup_one_client): - self.do_test_ok_preauth_and_bootstrap(standard_setup_one_client) - - def test_ok_preauth_and_remove(self, standard_setup_one_client): - self.do_test_ok_preauth_and_remove() - - def test_fail_preauth_existing(self, standard_setup_one_client): - self.do_test_fail_preauth_existing() - - -class TestPreauthEnterprise(TestPreauthBase): - def test_ok_preauth_and_bootstrap(self, enterprise_no_client): - self.__create_tenant_and_container(enterprise_no_client) - self.do_test_ok_preauth_and_bootstrap(enterprise_no_client) - - def test_ok_preauth_and_remove(self, enterprise_no_client): - self.__create_tenant_and_container(enterprise_no_client) - self.do_test_ok_preauth_and_remove() - - def test_fail_preauth_existing(self, enterprise_no_client): - self.__create_tenant_and_container(enterprise_no_client) - self.do_test_fail_preauth_existing() - - def __create_tenant_and_container(self, container_manager): - uuidv4 = str(uuid.uuid4()) - auth.new_tenant( - "test.mender.io-" + uuidv4, - "some.user+" + uuidv4 + "@example.com", - "hunter2hunter2", - ) - token = auth.current_tenant["tenant_token"] - - container_manager.new_tenant_client("tenant-container", token) - mender_device = MenderDevice(container_manager.get_mender_clients()[0]) - mender_device.ssh_is_opened() - container_manager.device = mender_device - - -class Client: - """Wraps various actions on the client, performed via SSH (inside fabric.execute()).""" - - ID_HELPER = "/usr/share/mender/identity/mender-device-identity" - PRIV_KEY = "/data/mender/mender-agent.pem" - - KEYGEN_TIMEOUT = 300 - - @staticmethod - def get_logs(device): - output_from_journalctl = device.run("journalctl --unit mender-updated --full") - logger.info(output_from_journalctl) - - @staticmethod - def get_pub_key(device): - """Extract the device's public key from its private key.""" - - Client.__wait_for_keygen(device) - keystr = device.run("cat {}".format(Client.PRIV_KEY)) - private_key = serialization.load_pem_private_key( - data=keystr.encode() if isinstance(keystr, str) else keystr, - password=None, - backend=default_backend(), - ) - public_key = private_key.public_key() - return public_key.public_bytes( - serialization.Encoding.PEM, serialization.PublicFormat.SubjectPublicKeyInfo - ).decode() - - @staticmethod - def substitute_id_data(device, id_data_dict): - """Change the device's identity by substituting it's id data helper script.""" - - id_data = "#!/bin/sh\n" - for k, v in id_data_dict.items(): - id_data += "echo {}={}\n".format(k, v) - - cmd = 'echo "{}" > {}'.format(id_data, Client.ID_HELPER) - device.run(cmd) - - @staticmethod - def __wait_for_keygen(device): - sleepsec = 0 - while sleepsec < Client.KEYGEN_TIMEOUT: - try: - device.run("stat {}".format(Client.PRIV_KEY)) - except: - time.sleep(10) - sleepsec += 10 - logger.info("waiting for key gen, sleepsec: {}".format(sleepsec)) - else: - time.sleep(5) - break - - assert sleepsec <= Client.KEYGEN_TIMEOUT, "timeout for key generation exceeded" diff --git a/tests/tests/test_provides_depends.py b/tests/tests/test_provides_depends.py deleted file mode 100644 index 04bf74958..000000000 --- a/tests/tests/test_provides_depends.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/python -# Copyright 2021 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import subprocess -import time -import uuid - -from ..common_setup import enterprise_no_client -from .common_update import update_image, common_update_procedure -from .mendertesting import MenderTesting -from ..MenderAPI import auth, devauth, deploy, logger - -from testutils.infra.device import MenderDevice - - -class TestProvidesDependsEnterprise(MenderTesting): - def test_update_provides_depends(self, enterprise_no_client): - """ - Perform two consecutive updates, the first adds virtual provides - to the artifact and the second artifact depends on these provides. - """ - uuidv4 = str(uuid.uuid4()) - tname = "test.mender.io-{}".format(uuidv4) - email = "some.user+{}@example.com".format(uuidv4) - - # Create tenant user - auth.reset_auth_token() - auth.new_tenant(tname, email, "secret-service", "enterprise") - token = auth.current_tenant["tenant_token"] - - # Create client setup with tenant token - enterprise_no_client.new_tenant_docker_client("mender-client", token) - mender_device = MenderDevice(enterprise_no_client.get_mender_clients()[0]) - - # Wait for ssh to be open - mender_device.ssh_is_opened() - # Check that the device has authorized with the backend. - devauth.get_devices(expected_devices=1) - devauth.accept_devices(1) - assert len(devauth.get_devices_status("accepted")) == 1 - - # Update client with and artifact with custom provides - def prepare_provides_artifact(artifact_file, artifact_id): - cmd = ( - # Package tests folder in the artifact, just a random folder. - "directory-artifact-gen -o %s -n %s -t docker-client -d /tmp/test_file_update_module tests -- --provides rootfs-image.directory.foo:bar" - % (artifact_file, artifact_id) - ) - logger.info("Executing: " + cmd) - subprocess.check_call(cmd, shell=True) - return artifact_file - - deployment_id, _ = common_update_procedure( - make_artifact=prepare_provides_artifact, - # We use verify_status=False, because update module updates are so - # quick that it sometimes races past the 'inprogress' status without - # the test framework having time to register it. That's not really - # the part we're interested in though, so just skip it. - verify_status=False, - ) - deploy.check_expected_status("finished", deployment_id) - - # Issue another update which depends on the custom provides - def prepare_depends_artifact(artifact_file, artifact_id): - cmd = ( - # Package tests folder in the artifact, just a random folder. - "directory-artifact-gen -o %s -n %s -t docker-client -d /tmp/test_file_update_module tests -- --depends rootfs-image.directory.foo:bar" - % (artifact_file, artifact_id) - ) - logger.info("Executing: " + cmd) - subprocess.check_call(cmd, shell=True) - return artifact_file - - deployment_id, _ = common_update_procedure( - make_artifact=prepare_depends_artifact, - verify_status=False, - ) - deploy.check_expected_status("finished", deployment_id) - - # Issue a third update with the same update as previous, this time - # with insufficient provides -> no artifact status - deployment_id, _ = common_update_procedure( - make_artifact=prepare_depends_artifact, verify_status=False - ) - - # Retry for at most 60 seconds checking for deployment status update - stat = None - noartifact = 0 - for i in range(60): - time.sleep(1) - stat = deploy.get_statistics(deployment_id) - if stat.get("noartifact") == 1: - noartifact = 1 - break - - assert stat is not None - assert noartifact == 1 diff --git a/tests/tests/test_security.py b/tests/tests/test_security.py deleted file mode 100644 index 9bf7ebe71..000000000 --- a/tests/tests/test_security.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2023 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import subprocess -import contextlib -import ssl -import socket -import time - -import pytest - -from ..common_setup import ( - running_custom_production_setup, - standard_setup_with_short_lived_token, - enterprise_with_short_lived_token, -) -from ..helpers import Helpers -from ..MenderAPI import DeviceAuthV2, Deployments, logger -from .common_update import common_update_procedure -from .mendertesting import MenderTesting - - -class BaseTestSecurity(MenderTesting): - def do_test_token_token_expiration(self, env, valid_image_with_mender_conf): - """verify that an expired token is handled correctly (client gets a new, valid one) - and that deployments are still received by the client - """ - mender_device = env.device - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - Helpers.check_log_is_authenticated(mender_device) - - # this call verifies that the deployment process goes into an "inprogress" state - # which is only possible when the client has a valid token. - mender_conf = mender_device.run("cat /etc/mender/mender.conf") - common_update_procedure( - install_image=valid_image_with_mender_conf(mender_conf), - devauth=devauth, - deploy=deploy, - ) - - -class TestSecurityOpenSource(BaseTestSecurity): - def test_token_token_expiration( - self, standard_setup_with_short_lived_token, valid_image_with_mender_conf - ): - self.do_test_token_token_expiration( - standard_setup_with_short_lived_token, valid_image_with_mender_conf - ) - - -class TestSecurityEnterprise(BaseTestSecurity): - def test_token_token_expiration( - self, enterprise_with_short_lived_token, valid_image_with_mender_conf - ): - self.do_test_token_token_expiration( - enterprise_with_short_lived_token, valid_image_with_mender_conf - ) diff --git a/tests/tests/test_signed_image_update.py b/tests/tests/test_signed_image_update.py deleted file mode 100644 index 1b51c9a16..000000000 --- a/tests/tests/test_signed_image_update.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2022 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest - -from ..common_setup import ( - standard_setup_with_signed_artifact_client, - enterprise_with_signed_artifact_client, -) -from .common_update import update_image, common_update_procedure -from ..MenderAPI import DeviceAuthV2, Deployments -from .mendertesting import MenderTesting - - -class BaseTestSignedUpdates(MenderTesting): - """ - Signed artifacts are well tested in the client's acceptance tests, so - we will only test basic backend integration with signed images here. - """ - - def do_test_signed_artifact_success(self, env, valid_image_with_mender_conf): - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - mender_conf = env.device.run("cat /etc/mender/mender.conf") - update_image( - env.device, - env.get_virtual_network_host_ip(), - install_image=valid_image_with_mender_conf(mender_conf), - signed=True, - devauth=devauth, - deploy=deploy, - ) - - def do_test_unsigned_artifact_fails_deployment( - self, env, valid_image_with_mender_conf - ): - """ - Make sure that an unsigned image fails, and is handled by the backend. - Notice that this test needs a fresh new version of the backend, since - we installed a signed image earlier without a verification key in mender.conf - """ - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - mender_conf = env.device.run("cat /etc/mender/mender.conf") - deployment_id, _ = common_update_procedure( - install_image=valid_image_with_mender_conf(mender_conf), - devauth=devauth, - deploy=deploy, - ) - deploy.check_expected_status("finished", deployment_id) - deploy.check_expected_statistics(deployment_id, "failure", 1) - - for d in devauth.get_devices(): - assert ( - "expecting signed artifact, but no signature file found" - in deploy.get_logs(d["id"], deployment_id) - ) - - -@MenderTesting.fast -class TestSignedUpdatesOpenSource(BaseTestSignedUpdates): - def test_signed_artifact_success( - self, standard_setup_with_signed_artifact_client, valid_image_with_mender_conf - ): - self.do_test_signed_artifact_success( - standard_setup_with_signed_artifact_client, valid_image_with_mender_conf - ) - - @pytest.mark.parametrize( - "standard_setup_with_signed_artifact_client", ["force_new"], indirect=True - ) - def test_unsigned_artifact_fails_deployment( - self, standard_setup_with_signed_artifact_client, valid_image_with_mender_conf - ): - self.do_test_unsigned_artifact_fails_deployment( - standard_setup_with_signed_artifact_client, valid_image_with_mender_conf - ) - - -@MenderTesting.fast -class TestSignedUpdatesEnterprise(BaseTestSignedUpdates): - def test_signed_artifact_success( - self, enterprise_with_signed_artifact_client, valid_image_with_mender_conf - ): - self.do_test_signed_artifact_success( - enterprise_with_signed_artifact_client, valid_image_with_mender_conf - ) - - @pytest.mark.parametrize( - "enterprise_with_signed_artifact_client", ["force_new"], indirect=True - ) - def test_unsigned_artifact_fails_deployment( - self, enterprise_with_signed_artifact_client, valid_image_with_mender_conf - ): - self.do_test_unsigned_artifact_fails_deployment( - enterprise_with_signed_artifact_client, valid_image_with_mender_conf - ) diff --git a/tests/tests/test_state_scripts.py b/tests/tests/test_state_scripts.py deleted file mode 100644 index ebcfd0121..000000000 --- a/tests/tests/test_state_scripts.py +++ /dev/null @@ -1,991 +0,0 @@ -# Copyright 2023 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import shutil -import time -import os -import subprocess -import pathlib - -import pytest - -from .. import conftest -from ..common_setup import ( - class_persistent_standard_setup_one_client_bootstrapped, - class_persistent_enterprise_one_client_bootstrapped, -) -from .common_update import common_update_procedure -from ..helpers import Helpers -from ..MenderAPI import DeviceAuthV2, Deployments, logger, image -from .mendertesting import MenderTesting -from testutils.infra.device import MenderDeviceGroup - - -@pytest.fixture(scope="class") -def class_persistent_setup_client_state_scripts_update_module( - class_persistent_standard_setup_one_client_bootstrapped, -): - device = class_persistent_standard_setup_one_client_bootstrapped.device - device.put( - "module-state-scripts-test", - local_path=pathlib.Path(__file__).parent.parent.absolute(), - remote_path="/usr/share/mender/modules/v3", - ) - - return class_persistent_standard_setup_one_client_bootstrapped - - -@pytest.fixture(scope="class") -def class_persistent_enterprise_setup_client_state_scripts_update_module( - class_persistent_enterprise_one_client_bootstrapped, -): - device = class_persistent_enterprise_one_client_bootstrapped.device - device.put( - "module-state-scripts-test", - local_path=pathlib.Path(__file__).parent.parent.absolute(), - remote_path="/usr/share/mender/modules/v3", - ) - - return class_persistent_enterprise_one_client_bootstrapped - - -TEST_SETS = [ - ( - "Normal_success", - { - "FailureScript": [], - "ExpectedStatus": "success", - "ScriptOrder": [ - "Idle_Enter_08_testing", - "Idle_Enter_09", - "Idle_Leave_09", - "Idle_Leave_10", - "Sync_Enter_02", - "Sync_Enter_03", - "Sync_Leave_04", - "Sync_Leave_15", - "Download_Enter_12", - "Download_Enter_13", - "Download_Leave_14", - "Download_Leave_25", - "ArtifactInstall_Enter_01", - "ArtifactInstall_Enter_02", - "ArtifactInstall_Leave_01", - "ArtifactInstall_Leave_03", - "ArtifactReboot_Enter_01", - "ArtifactReboot_Enter_11", - "ArtifactReboot_Leave_01", - "ArtifactReboot_Leave_89", - "ArtifactReboot_Leave_99", - "ArtifactCommit_Enter_01", - "ArtifactCommit_Enter_05", - "ArtifactCommit_Leave_01_extra_string", - "ArtifactCommit_Leave_91", - ], - }, - ), - ( - "Failure_in_Idle_Enter_script", - { - "FailureScript": ["Idle_Enter_09"], - "ExpectedStatus": "success", - "ScriptOrder": [ - "Idle_Enter_08_testing", - "Idle_Enter_09", # Error in this script should not have any effect. - "Idle_Leave_09", - "Idle_Leave_10", - "Sync_Enter_02", - "Sync_Enter_03", - "Sync_Leave_04", - "Sync_Leave_15", - "Download_Enter_12", - "Download_Enter_13", - "Download_Leave_14", - "Download_Leave_25", - "ArtifactInstall_Enter_01", - "ArtifactInstall_Enter_02", - "ArtifactInstall_Leave_01", - "ArtifactInstall_Leave_03", - "ArtifactReboot_Enter_01", - "ArtifactReboot_Enter_11", - "ArtifactReboot_Leave_01", - "ArtifactReboot_Leave_89", - "ArtifactReboot_Leave_99", - "ArtifactCommit_Enter_01", - "ArtifactCommit_Enter_05", - "ArtifactCommit_Leave_01_extra_string", - "ArtifactCommit_Leave_91", - ], - }, - ), - ( - "Failure_in_Idle_Leave_script", - { - "FailureScript": ["Idle_Leave_09"], - "ExpectedStatus": "success", - "ScriptOrder": [ - "Idle_Enter_08_testing", - "Idle_Enter_09", - "Idle_Leave_09", # Error in this script should not have any effect. - "Idle_Leave_10", - "Sync_Enter_02", - "Sync_Enter_03", - "Sync_Leave_04", - "Sync_Leave_15", - "Download_Enter_12", - "Download_Enter_13", - "Download_Leave_14", - "Download_Leave_25", - "ArtifactInstall_Enter_01", - "ArtifactInstall_Enter_02", - "ArtifactInstall_Leave_01", - "ArtifactInstall_Leave_03", - "ArtifactReboot_Enter_01", - "ArtifactReboot_Enter_11", - "ArtifactReboot_Leave_01", - "ArtifactReboot_Leave_89", - "ArtifactReboot_Leave_99", - "ArtifactCommit_Enter_01", - "ArtifactCommit_Enter_05", - "ArtifactCommit_Leave_01_extra_string", - "ArtifactCommit_Leave_91", - ], - }, - ), - ( - "Failure_in_Sync_Enter_script", - { - "FailureScript": ["Sync_Enter_02"], - "ExpectedStatus": None, - "ScriptOrder": [ - "Idle_Enter_08_testing", - "Idle_Enter_09", - "Idle_Leave_09", - "Idle_Leave_10", - "Sync_Enter_02", - "Sync_Error_15", - "Sync_Error_16", - ], - }, - ), - ( - "Failure_in_Sync_Leave_script", - { - "FailureScript": ["Sync_Leave_15"], - "ExpectedStatus": None, - "ScriptOrder": [ - "Idle_Enter_08_testing", - "Idle_Enter_09", - "Idle_Leave_09", - "Idle_Leave_10", - "Sync_Enter_02", - "Sync_Enter_03", - "Sync_Leave_04", - "Sync_Leave_15", - "Sync_Error_15", - "Sync_Error_16", - ], - }, - ), - ( - "Failure_in_Download_Enter_script", - { - "FailureScript": ["Download_Enter_12"], - "ExpectedStatus": "failure", - "ScriptOrder": [ - "Idle_Enter_08_testing", - "Idle_Enter_09", - "Idle_Leave_09", - "Idle_Leave_10", - "Sync_Enter_02", - "Sync_Enter_03", - "Sync_Leave_04", - "Sync_Leave_15", - "Download_Enter_12", - "Download_Error_25", - ], - }, - ), - ( - "Failure_in_Download_Leave_script", - { - "FailureScript": ["Download_Leave_14"], - "ExpectedStatus": "failure", - "ScriptOrder": [ - "Idle_Enter_08_testing", - "Idle_Enter_09", - "Idle_Leave_09", - "Idle_Leave_10", - "Sync_Enter_02", - "Sync_Enter_03", - "Sync_Leave_04", - "Sync_Leave_15", - "Download_Enter_12", - "Download_Enter_13", - "Download_Leave_14", - "Download_Error_25", - ], - }, - ), - ( - "Failure_in_ArtifactInstall_Enter_script", - { - "FailureScript": ["ArtifactInstall_Enter_01"], - "ExpectedStatus": "failure", - "ScriptOrder": [ - "Idle_Enter_08_testing", - "Idle_Enter_09", - "Idle_Leave_09", - "Idle_Leave_10", - "Sync_Enter_02", - "Sync_Enter_03", - "Sync_Leave_04", - "Sync_Leave_15", - "Download_Enter_12", - "Download_Enter_13", - "Download_Leave_14", - "Download_Leave_25", - "ArtifactInstall_Enter_01", - "ArtifactInstall_Error_01", - "ArtifactInstall_Error_02", - "ArtifactInstall_Error_99", - "ArtifactRollback_Enter_00", - "ArtifactRollback_Enter_01", - "ArtifactRollback_Leave_00", - "ArtifactRollback_Leave_01", - "ArtifactRollbackReboot_Enter_00", - "ArtifactRollbackReboot_Enter_99", - "ArtifactRollbackReboot_Leave_01", - "ArtifactRollbackReboot_Leave_99", - "ArtifactFailure_Enter_22", - "ArtifactFailure_Enter_33", - "ArtifactFailure_Leave_44", - "ArtifactFailure_Leave_55", - ], - }, - ), - ( - "Failure_in_ArtifactCommit_Enter_script", - { - "FailureScript": ["ArtifactCommit_Enter_05"], - "ExpectedStatus": "failure", - "ScriptOrder": [ - "Idle_Enter_08_testing", - "Idle_Enter_09", - "Idle_Leave_09", - "Idle_Leave_10", - "Sync_Enter_02", - "Sync_Enter_03", - "Sync_Leave_04", - "Sync_Leave_15", - "Download_Enter_12", - "Download_Enter_13", - "Download_Leave_14", - "Download_Leave_25", - "ArtifactInstall_Enter_01", - "ArtifactInstall_Enter_02", - "ArtifactInstall_Leave_01", - "ArtifactInstall_Leave_03", - "ArtifactReboot_Enter_01", - "ArtifactReboot_Enter_11", - "ArtifactReboot_Leave_01", - "ArtifactReboot_Leave_89", - "ArtifactReboot_Leave_99", - "ArtifactCommit_Enter_01", - "ArtifactCommit_Enter_05", - "ArtifactCommit_Error_91", - "ArtifactRollback_Enter_00", - "ArtifactRollback_Enter_01", - "ArtifactRollback_Leave_00", - "ArtifactRollback_Leave_01", - "ArtifactRollbackReboot_Enter_00", - "ArtifactRollbackReboot_Enter_99", - "ArtifactRollbackReboot_Leave_01", - "ArtifactRollbackReboot_Leave_99", - "ArtifactFailure_Enter_22", - "ArtifactFailure_Enter_33", - "ArtifactFailure_Leave_44", - "ArtifactFailure_Leave_55", - ], - }, - ), - ( - "Failure_in_ArtifactCommit_Leave_script", - { - "FailureScript": ["ArtifactCommit_Leave_01_extra_string"], - "ExpectedStatus": "failure", - "SwapPartitionExpectation": True, - "ScriptOrder": [ - "Idle_Enter_08_testing", - "Idle_Enter_09", - "Idle_Leave_09", - "Idle_Leave_10", - "Sync_Enter_02", - "Sync_Enter_03", - "Sync_Leave_04", - "Sync_Leave_15", - "Download_Enter_12", - "Download_Enter_13", - "Download_Leave_14", - "Download_Leave_25", - "ArtifactInstall_Enter_01", - "ArtifactInstall_Enter_02", - "ArtifactInstall_Leave_01", - "ArtifactInstall_Leave_03", - "ArtifactReboot_Enter_01", - "ArtifactReboot_Enter_11", - "ArtifactReboot_Leave_01", - "ArtifactReboot_Leave_89", - "ArtifactReboot_Leave_99", - "ArtifactCommit_Enter_01", - "ArtifactCommit_Enter_05", - "ArtifactCommit_Leave_01_extra_string", # Error in this script should not have any effect. - "ArtifactCommit_Error_91", - ], - }, - ), - ( - "Corrupted_script_version_in_data", - { - "FailureScript": [], - "ExpectedStatus": "failure", - "CorruptDataScriptVersionIn": "ArtifactReboot_Enter_11", - "ScriptOrder": [ - "Idle_Enter_08_testing", - "Idle_Enter_09", - "Idle_Leave_09", - "Idle_Leave_10", - "Sync_Enter_02", - "Sync_Enter_03", - "Sync_Leave_04", - "Sync_Leave_15", - "Download_Enter_12", - "Download_Enter_13", - "Download_Leave_14", - "Download_Leave_25", - "ArtifactInstall_Enter_01", - "ArtifactInstall_Enter_02", - "ArtifactInstall_Leave_01", - "ArtifactInstall_Leave_03", - "ArtifactReboot_Enter_01", - "ArtifactReboot_Enter_11", - # since version is corrupted from now on, no more scripts - # will be executed, but rollback will be performed - ], - }, - ), - ( - "Corrupted_script_version_in_etc", - { - "FailureScript": [], - "ExpectedStatus": "failure", - "CorruptEtcScriptVersionIn": "ArtifactReboot_Leave_99", - "RestoreEtcScriptVersionIn": "ArtifactRollbackReboot_Leave_99", - "ScriptOrder": [ - "Idle_Enter_08_testing", - "Idle_Enter_09", - "Idle_Leave_09", - "Idle_Leave_10", - "Sync_Enter_02", - "Sync_Enter_03", - "Sync_Leave_04", - "Sync_Leave_15", - "Download_Enter_12", - "Download_Enter_13", - "Download_Leave_14", - "Download_Leave_25", - "ArtifactInstall_Enter_01", - "ArtifactInstall_Enter_02", - "ArtifactInstall_Leave_01", - "ArtifactInstall_Leave_03", - "ArtifactReboot_Enter_01", - "ArtifactReboot_Enter_11", - "ArtifactReboot_Leave_01", - "ArtifactReboot_Leave_89", - "ArtifactReboot_Leave_99", - "ArtifactCommit_Enter_01", - "ArtifactCommit_Enter_05", - "ArtifactCommit_Error_91", - "ArtifactRollback_Enter_00", - "ArtifactRollback_Enter_01", - "ArtifactRollback_Leave_00", - "ArtifactRollback_Leave_01", - "ArtifactRollbackReboot_Enter_00", - "ArtifactRollbackReboot_Enter_99", - "ArtifactRollbackReboot_Leave_01", - "ArtifactRollbackReboot_Leave_99", - "ArtifactFailure_Enter_22", - "ArtifactFailure_Enter_33", - "ArtifactFailure_Leave_44", - "ArtifactFailure_Leave_55", - ], - }, - ), -] - - -REBOOT_TEST_SET = [ - ( - "simulate_powerloss_artifact_install_enter", - { - "RebootScripts": ["ArtifactInstall_Enter_02"], - "ExpectedFinalPartition": ["OriginalPartition"], - "ScriptOrder": [ - "ArtifactInstall_Enter_01", - "ArtifactInstall_Enter_02", - "ArtifactInstall_Leave_01", - "ArtifactReboot_Enter_01", - "ArtifactReboot_Leave_01", - "ArtifactFailure_Enter_01", - "ArtifactFailure_Leave_89", - ], - "ExpectedScriptFlow": [ - "ArtifactInstall_Enter_01", # run one script to init log - "ArtifactInstall_Enter_02", # kill! - "ArtifactFailure_Enter_01", # run failure scripts - "ArtifactFailure_Leave_89", - ], - }, - ), - ( - "simulate_powerloss_in_commit_enter", - { - "RebootScripts": ["ArtifactCommit_Enter_89"], - "ExpectedFinalPartition": ["OriginalPartition"], - "ScriptOrder": [ - "ArtifactInstall_Enter_01", - "ArtifactInstall_Leave_01", - "ArtifactReboot_Enter_01", - "ArtifactReboot_Leave_01", - "ArtifactCommit_Enter_89", - "ArtifactRollback_Enter_00", - "ArtifactRollbackReboot_Enter_89", - "ArtifactFailure_Enter_89", - "ArtifactFailure_Leave_09", - ], - "ExpectedScriptFlow": [ - "ArtifactInstall_Enter_01", - "ArtifactInstall_Leave_01", - "ArtifactReboot_Enter_01", - "ArtifactReboot_Leave_01", # on second partition, stop mender client - "ArtifactCommit_Enter_89", # sync and kill! - "ArtifactRollback_Enter_00", - "ArtifactRollbackReboot_Enter_89", - "ArtifactFailure_Enter_89", # run failure scripts on the committed (old) partition - "ArtifactFailure_Leave_09", - ], - }, - ), - ( - "simulate_powerloss_in_artifact_commit_leave", - { - "RebootOnceScripts": ["ArtifactCommit_Leave_01"], - "ExpectedFinalPartition": ["OtherPartition"], - "ScriptOrder": [ - "ArtifactInstall_Enter_01", - "ArtifactInstall_Leave_01", - "ArtifactReboot_Enter_01", - "ArtifactReboot_Leave_01", - "ArtifactCommit_Enter_89", - "ArtifactCommit_Leave_01", - "ArtifactCommit_Leave_02", - ], - "ExpectedScriptFlow": [ - "ArtifactInstall_Enter_01", - "ArtifactInstall_Leave_01", - "ArtifactReboot_Enter_01", - "ArtifactReboot_Leave_01", - "ArtifactCommit_Enter_89", - "ArtifactCommit_Leave_01", # kill! - "ArtifactCommit_Leave_01", # rerun - "ArtifactCommit_Leave_02", - ], - }, - ), -] - - -class BaseTestStateScripts(MenderTesting): - scripts = [ - "Idle_Enter_08_testing", - "Idle_Enter_09", - "Idle_Enter_100", # Invalid script, should never be run. - "Idle_Leave_09", - "Idle_Leave_10", - "Idle_Error_00", - "Sync_Enter_02", - "Sync_Enter_03", - "Sync_Leave_04", - "Sync_Leave_15", - "Sync_Error_15", - "Sync_Error_16", - "Download_Enter_12", - "Download_Enter_13", - "Download_Leave_14", - "Download_Leave_25", - "Download_Error_25", - "ArtifactInstall_Enter_01", - "ArtifactInstall_Enter_02", - "ArtifactInstall_Leave_01", - "ArtifactInstall_Leave_03", - "ArtifactInstall_Error_01", - "ArtifactInstall_Error_02", - "ArtifactInstall_Error_99", - "ArtifactReboot_Enter_01", - "ArtifactReboot_Enter_11", - "ArtifactReboot_Leave_01", - "ArtifactReboot_Leave_89", - "ArtifactReboot_Leave_99", - "ArtifactReboot_Error_97", - "ArtifactReboot_Error_98", - "ArtifactCommit_Enter_01", - "ArtifactCommit_Enter_05", - "ArtifactCommit_Leave_01_extra_string", - "ArtifactCommit_Leave_91", - "ArtifactCommit_Error_91", - "ArtifactRollback_Enter_00", - "ArtifactRollback_Enter_01", - "ArtifactRollback_Leave_00", - "ArtifactRollback_Leave_01", - "ArtifactRollback_Error_15", # Error for this state doesn't exist, should never run. - "ArtifactRollbackReboot_Enter_00", - "ArtifactRollbackReboot_Enter_99", - "ArtifactRollbackReboot_Leave_01", - "ArtifactRollbackReboot_Leave_99", - "ArtifactRollbackReboot_Error_88", # Error for this state doesn't exist, should never run. - "ArtifactRollbackReboot_Error_99", # Error for this state doesn't exist, should never run. - "ArtifactFailure_Enter_22", - "ArtifactFailure_Enter_33", - "ArtifactFailure_Leave_44", - "ArtifactFailure_Leave_55", - "ArtifactFailure_Error_55", # Error for this state doesn't exist, should never run. - ] - - def do_test_reboot_recovery( - self, - env, - description, - test_set, - ): - - mender_device = env.device - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - work_dir = "test_state_scripts.%s" % mender_device.host_string - - script_content = ( - '#!/bin/sh\n\necho "$(basename $0)" >> /data/test_state_scripts.log\n' - ) - - script_failure_content = ( - script_content + "sync\necho b > /proc/sysrq-trigger\n" - ) # flush to disk before killing - - # This is only needed in the case: die commit-leave, - # otherwise the device will get stuck in a boot-reboot loop - script_reboot_once = """#!/bin/sh - if [ $(grep -c $(basename $0) /data/test_state_scripts.log) -eq 0 ]; then - echo "$(basename $0)" >> /data/test_state_scripts.log && sync && echo b > /proc/sysrq-trigger - fi - echo "$(basename $0)" >> /data/test_state_scripts.log - exit 0""" - - # Put artifact-scripts in the artifact. - artifact_script_dir = os.path.join(work_dir, "artifact-scripts") - - if os.path.exists(work_dir): - shutil.rmtree(work_dir, ignore_errors=True) - - os.mkdir(work_dir) - os.mkdir(artifact_script_dir) - - for script in test_set.get("ScriptOrder"): - if not script.startswith("Artifact"): - # Not an artifact script, skip this one. - continue - with open(os.path.join(artifact_script_dir, script), "w") as fd: - if script in test_set.get("RebootScripts", []): - fd.write(script_failure_content) - if script in test_set.get("RebootOnceScripts", []): - fd.write(script_reboot_once) - else: - fd.write(script_content) - - # Now create the artifact, and make the deployment. - device_id = Helpers.ip_to_device_id_map( - MenderDeviceGroup([mender_device.host_string]), - devauth=devauth, - )[mender_device.host_string] - - host_ip = env.get_virtual_network_host_ip() - - def make_artifact(filename, artifact_name): - return image.make_module_artifact( - "module-state-scripts-test", - conftest.machine_name, - artifact_name, - filename, - scripts=[artifact_script_dir], - ) - - with mender_device.get_reboot_detector(host_ip) as reboot_detector: - - common_update_procedure( - verify_status=True, - devices=[device_id], - scripts=[artifact_script_dir], - make_artifact=make_artifact, - devauth=devauth, - deploy=deploy, - ) - - try: - reboot_detector.verify_reboot_performed() - - # wait until the last script has been run - logger.debug("Wait until the last script has been run") - script_logs = "" - timeout = time.time() + 10 * 60 - while timeout >= time.time(): - time.sleep(3) - try: - script_logs = mender_device.run( - "cat /data/test_state_scripts.log" - ) - if test_set.get("ExpectedScriptFlow")[-1] in script_logs: - break - except EOFError: - # In some cases the SSH connection raises here EOF due to the - # client simulating powerloss. The test will just retry - pass - else: - pytest.fail( - "Timeout waiting for ExpectedScriptFlow in state scripts. Expected %s, got %s" - % ( - test_set.get("ExpectedScriptFlow"), - ", ".join(script_logs.rstrip().split("\n")), - ) - ) - - assert script_logs.split() == test_set.get("ExpectedScriptFlow") - - except: - output = mender_device.run( - "cat /data/mender/deployment*.log", warn_only=True - ) - logger.info(output) - raise - - finally: - mender_device.run( - "systemctl stop mender-updated && " - + "rm -f /data/test_state_scripts.log && " - + "rm -rf /etc/mender/scripts && " - + "rm -rf /data/mender/scripts && " - + "systemctl start mender-updated" - ) - - def do_test_state_scripts( - self, - env, - description, - test_set, - ): - """Test that state scripts are executed in right order, and that errors - are treated like they should.""" - - mender_device = env.device - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - work_dir = "test_state_scripts.%s" % mender_device.host_string - deployment_id = None - try: - script_content = '#!/bin/sh\n\necho "`date --rfc-3339=seconds` $(basename $0)" >> /data/test_state_scripts.log\n' - script_failure_content = script_content + "exit 1\n" - - # Make rootfs-scripts and put them in rootfs image. - rootfs_script_dir = os.path.join(work_dir, "rootfs-scripts") - shutil.rmtree(work_dir, ignore_errors=True) - os.mkdir(work_dir) - os.mkdir(rootfs_script_dir) - - for script in self.scripts: - if script.startswith("Artifact"): - # This is a script for the artifact, skip this one. - continue - with open(os.path.join(rootfs_script_dir, script), "w") as fd: - if script in test_set["FailureScript"]: - fd.write(script_failure_content) - else: - fd.write(script_content) - os.fchmod(fd.fileno(), 0o0755) - - # Write this again in case it was corrupted above. - with open(os.path.join(rootfs_script_dir, "version"), "w") as fd: - fd.write("3") - - # Then zip and copy them to QEMU host. - subprocess.check_call( - ["tar", "czf", "../rootfs-scripts.tar.gz", "."], cwd=rootfs_script_dir - ) - # Stop client first to avoid race conditions. - mender_device.run("systemctl stop mender-updated") - try: - mender_device.put( - os.path.join(work_dir, "rootfs-scripts.tar.gz"), remote_path="/" - ) - mender_device.run( - "mkdir -p cd /etc/mender/scripts && " - + "cd /etc/mender/scripts && " - + "tar xzf /rootfs-scripts.tar.gz && " - + "rm -f /rootfs-scripts.tar.gz" - ) - finally: - mender_device.run("systemctl start mender-updated") - - # Put artifact-scripts in the artifact. - artifact_script_dir = os.path.join(work_dir, "artifact-scripts") - os.mkdir(artifact_script_dir) - for script in self.scripts: - if not script.startswith("Artifact"): - # Not an artifact script, skip this one. - continue - with open(os.path.join(artifact_script_dir, script), "w") as fd: - if script in test_set["FailureScript"]: - fd.write(script_failure_content) - else: - fd.write(script_content) - if test_set.get("CorruptDataScriptVersionIn") == script: - fd.write("printf '1000' > /data/mender/scripts/version\n") - if test_set.get("CorruptEtcScriptVersionIn") == script: - fd.write("printf '1000' > /etc/mender/scripts/version\n") - if test_set.get("RestoreEtcScriptVersionIn") == script: - fd.write("printf '3' > /etc/mender/scripts/version\n") - - # Callback for our custom artifact maker - def make_artifact(filename, artifact_name): - return image.make_module_artifact( - "module-state-scripts-test", - conftest.machine_name, - artifact_name, - filename, - scripts=[artifact_script_dir], - ) - - # Now create the artifact, and make the deployment. - device_id = Helpers.ip_to_device_id_map( - MenderDeviceGroup([mender_device.host_string]), - devauth=devauth, - )[mender_device.host_string] - deployment_id = common_update_procedure( - verify_status=False, - devices=[device_id], - scripts=[artifact_script_dir], - make_artifact=make_artifact, - devauth=devauth, - deploy=deploy, - )[0] - if test_set["ExpectedStatus"] is None: - # In this case we don't expect the deployment to even be - # attempted, presumably due to failing Idle/Sync/Download - # scripts on the client. So no deployment checking. Just wait - # until there is at least one Error script in the log, which - # will always be the case if ExpectedStatus is none (since one - # of them is preventing the update from being attempted). - def fetch_info(cmd_list): - all_output = "" - for cmd in cmd_list: - output = mender_device.run(cmd, warn_only=True) - logger.error("%s:\n%s" % (cmd, output)) - all_output += "%s\n" % output - return all_output - - info_query = [ - "cat /data/test_state_scripts.log 1>&2", - "journalctl --unit mender-updated", - "top -n5 -b", - "ls -l /proc/`pgrep mender-update`/fd", - "for fd in /proc/`pgrep mender-update`/fdinfo/*; do echo $fd:; cat $fd; done", - ] - starttime = time.time() - while starttime + 10 * 60 >= time.time(): - output = mender_device.run( - "grep Error /data/test_state_scripts.log", warn_only=True - ) - if output.rstrip() != "": - # If it succeeds, stop. - break - else: - fetch_info(info_query) - time.sleep(10) - continue - else: - info = fetch_info(info_query) - pytest.fail( - 'Waited too long for "Error" to appear in log:\n%s' % info - ) - else: - deploy.check_expected_statistics( - deployment_id, test_set["ExpectedStatus"], 1 - ) - - # Always give the client a little bit of time to settle in the base - # state after an update. - time.sleep(10) - - output = mender_device.run("cat /data/test_state_scripts.log") - self.verify_script_log_correct(test_set, output.split("\n")) - - except: - output = mender_device.run( - "cat /data/mender/deployment*.log", warn_only=True - ) - logger.info(output) - raise - - finally: - shutil.rmtree(work_dir, ignore_errors=True) - if deployment_id: - try: - deploy.abort(deployment_id) - except: - pass - mender_device.run( - "systemctl stop mender-updated && " - + "rm -f /data/test_state_scripts.log && " - + "rm -rf /etc/mender/scripts && " - + "rm -rf /data/mender/scripts && " - + "systemctl start mender-updated" - ) - - def verify_script_log_correct(self, test_set, log_orig): - expected_order = test_set["ScriptOrder"] - - # First remove timestamps from the log - log = [l.split(" ")[-1] for l in log_orig] - - # Iterate down the list of expected scripts, and make sure that the log - # follows the same list. - - # Position in log list. - log_pos = 0 - # Position in script list from test_set. - expected_pos = 0 - # Iterations around the full expected_order list - num_iterations = 0 - try: - while log_pos < len(log): - - if len(log[log_pos]) > 0: - # Make sure we are at right script. - assert expected_order[expected_pos] == log[log_pos] - - log_pos = log_pos + 1 - expected_pos = expected_pos + 1 - - if expected_pos == len(expected_order): - # We completed the expected sequence, we count as one full iteration - # and restart the index for the next iteration - num_iterations = num_iterations + 1 - expected_pos = 0 - - if ( - log_pos < len(log) - and log[log_pos - 1].startswith("Sync_") - and log[log_pos].startswith("Idle_") - and not expected_order[expected_pos].startswith("Idle_") - ): - # The Idle/Sync sequence is allowed to "wrap around" and start - # over, because it may take a few rounds of checking before the - # deployment is ready for the client. - expected_pos = 0 - - # Test cases with an expectation of success/failure shall do only 1 iteration - # Test cases with None expectation will loop through the error sequence in a loop, but still - # we want to make sure that it is reasonable. - # For these cases we set a min. of 4 iterations to ensure that the state machine is - # indeed looping and a max. of of 50 iterations to accommodate for slow running of the - # framework. - if test_set["ExpectedStatus"] is not None: - assert num_iterations == 1 - else: - assert num_iterations > 4 and num_iterations < 50 - - except: - logger.error( - "Exception in verify_script_log_correct: log of scripts = '%s'" - % "\n".join(log_orig) - ) - logger.error("scripts we expected = '%s'" % "\n".join(expected_order)) - raise - - -class TestStateScriptsOpenSource(BaseTestStateScripts): - @pytest.mark.parametrize("description,test_set", REBOOT_TEST_SET) - def test_reboot_recovery( - self, - class_persistent_setup_client_state_scripts_update_module, - description, - test_set, - ): - self.do_test_reboot_recovery( - class_persistent_setup_client_state_scripts_update_module, - description, - test_set, - ) - - @MenderTesting.slow - @pytest.mark.parametrize("description,test_set", TEST_SETS) - def test_state_scripts( - self, - class_persistent_setup_client_state_scripts_update_module, - description, - test_set, - ): - self.do_test_state_scripts( - class_persistent_setup_client_state_scripts_update_module, - description, - test_set, - ) - - -class TestStateScriptsEnterprise(BaseTestStateScripts): - @pytest.mark.parametrize("description,test_set", REBOOT_TEST_SET) - def test_reboot_recovery( - self, - class_persistent_enterprise_setup_client_state_scripts_update_module, - description, - test_set, - ): - self.do_test_reboot_recovery( - class_persistent_enterprise_setup_client_state_scripts_update_module, - description, - test_set, - ) - - @MenderTesting.slow - @pytest.mark.parametrize("description,test_set", TEST_SETS) - def test_state_scripts( - self, - class_persistent_enterprise_setup_client_state_scripts_update_module, - description, - test_set, - ): - self.do_test_state_scripts( - class_persistent_enterprise_setup_client_state_scripts_update_module, - description, - test_set, - ) diff --git a/tests/tests/test_tcp_teardown.py b/tests/tests/test_tcp_teardown.py deleted file mode 100644 index 2e180131a..000000000 --- a/tests/tests/test_tcp_teardown.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2023 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import json -import os.path -import tempfile -import time - -from ..common_setup import standard_setup_one_client -from ..MenderAPI import devauth, logger - - -def set_long_poll_intervals(mender_device): - with tempfile.NamedTemporaryFile(mode="w") as tf: - output = mender_device.run("cat /etc/mender/mender.conf") - config = json.loads(output) - config["InventoryPollIntervalSeconds"] = 3600 - config["UpdatePollIntervalSeconds"] = 3600 - json.dump(config, tf) - tf.flush() - mender_device.put( - os.path.basename(tf.name), - local_path=os.path.dirname(tf.name), - remote_path="/etc/mender/mender.conf", - ) - output = mender_device.run("cat /etc/mender/mender.conf") - logger.warning(output) - - -def get_opened_tcp_connections(mender_device, binary_name): - # First probe that a process for binary_name exists - mender_device.run(f"pidof {binary_name}") - output = mender_device.run( - f"for pid in `pidof {binary_name}`;" - + "do cat /proc/$pid/net/tcp;" - + "done" - + "| grep -E '[^:]+: [^ ]+ [^ ]+:01BB'" - + "| wc -l" - ) - return int(output) - - -class BaseTestTcpTeardown: - def test_tcp_teardown(self, standard_setup_one_client): - """Tests the closing of TCP sockets after HTTP requests on mender-auth and mender-update""" - mender_device = standard_setup_one_client.device - - # Stop all services - mender_device.run("systemctl stop mender-connect mender-authd mender-updated") - - # To verify mender-authd, trigger manually a token fetch - mender_device.run("systemctl start mender-authd") - time.sleep(5) - mender_device.run("""dbus-send --print-reply --system \\ - --dest=io.mender.AuthenticationManager \\ - /io/mender/AuthenticationManager \\ - io.mender.Authentication1.FetchJwtToken""") - # The fetch is done async, give it some time to finish - time.sleep(1) - assert get_opened_tcp_connections(mender_device, "mender-auth") == 0 - - # Accept the device and repeat the test. It should not make a difference - devauth.accept_devices(1) - mender_device.run("""dbus-send --print-reply --system \\ - --dest=io.mender.AuthenticationManager \\ - /io/mender/AuthenticationManager \\ - io.mender.Authentication1.FetchJwtToken""") - time.sleep(1) - assert get_opened_tcp_connections(mender_device, "mender-auth") == 0 - - # To test mender-update, set long intervals and manually trigger operations - set_long_poll_intervals(mender_device) - mender_device.run("systemctl start mender-updated") - time.sleep(5) - assert get_opened_tcp_connections(mender_device, "mender-update") == 0 - assert get_opened_tcp_connections(mender_device, "mender-auth") == 0 - - mender_device.run("mender-update check-update") - time.sleep(1) - assert get_opened_tcp_connections(mender_device, "mender-update") == 0 - assert get_opened_tcp_connections(mender_device, "mender-auth") == 0 - - mender_device.run("mender-update send-inventory") - time.sleep(1) - assert get_opened_tcp_connections(mender_device, "mender-update") == 0 - assert get_opened_tcp_connections(mender_device, "mender-auth") == 0 - - -class TestTcpTeardownOpenSource(BaseTestTcpTeardown): - pass - - -class TestTcpTeardownEnterprise(BaseTestTcpTeardown): - pass diff --git a/tests/tests/test_update_modules.py b/tests/tests/test_update_modules.py deleted file mode 100644 index 4a28eeef4..000000000 --- a/tests/tests/test_update_modules.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2022 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import subprocess -import tempfile -import shutil - -from .. import conftest -from ..common_setup import ( - standard_setup_one_docker_client_bootstrapped, - enterprise_one_docker_client_bootstrapped, -) -from .common_update import common_update_procedure -from ..MenderAPI import DeviceAuthV2, Deployments, logger -from .mendertesting import MenderTesting - - -class BaseTestUpdateModules(MenderTesting): - def do_test_rootfs_image_rejected(self, env): - """Test that a update for a non-existing module is rejected when such a setup isn't - present.""" - - mender_device = env.device - devauth = DeviceAuthV2(env.auth) - deploy = Deployments(env.auth, devauth) - - file_tree = tempfile.mkdtemp() - try: - file1 = os.path.join(file_tree, "file1") - with open(file1, "w") as fd: - fd.write("dummy") - - def make_artifact(artifact_file, artifact_id): - cmd = ( - "mender-artifact write module-image " - + "-o %s -n %s -t generic-x86_64 -T nonexisting-module -f %s" - % (artifact_file, artifact_id, file1) - ) - logger.info("Executing: " + cmd) - subprocess.check_call(cmd, shell=True) - return artifact_file - - deployment_id, _ = common_update_procedure( - make_artifact=make_artifact, devauth=devauth, deploy=deploy - ) - deploy.check_expected_status("finished", deployment_id) - deploy.check_expected_statistics(deployment_id, "failure", 1) - - output = mender_device.run("mender-update show-artifact").strip() - assert output == "original" - - output = env.get_logs_of_service("mender-client") - assert "Update Module not found for given artifact type" in output - assert ( - "Cannot launch /usr/share/mender/modules/v3/nonexisting-module" - in output - ) - - finally: - shutil.rmtree(file_tree) - - -class TestUpdateModulesOpenSource(BaseTestUpdateModules): - @MenderTesting.fast - def test_rootfs_image_rejected(self, standard_setup_one_docker_client_bootstrapped): - self.do_test_rootfs_image_rejected( - standard_setup_one_docker_client_bootstrapped - ) - - -class TestUpdateModulesEnterprise(BaseTestUpdateModules): - @MenderTesting.fast - def test_rootfs_image_rejected(self, enterprise_one_docker_client_bootstrapped): - self.do_test_rootfs_image_rejected(enterprise_one_docker_client_bootstrapped) From 21a4420628e204376f0fbc8272d314b6ef1e1899 Mon Sep 17 00:00:00 2001 From: Peter Grzybowski Date: Wed, 20 May 2026 09:40:49 +0200 Subject: [PATCH 3/8] chore: debug Ticket: QA-1625 Signed-off-by: Peter Grzybowski --- tests/tests/test_mender_connect.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/tests/test_mender_connect.py b/tests/tests/test_mender_connect.py index e0f57d748..b88675874 100644 --- a/tests/tests/test_mender_connect.py +++ b/tests/tests/test_mender_connect.py @@ -198,6 +198,7 @@ def test_bogus_shell_message(self, docker_env): def test_in_poor_network_environment(self, docker_env): self.assert_env(docker_env) + bp(0) receive_timeout_s = 16 @@ -231,6 +232,7 @@ def detect_shell_prompt(shell): docker_env.device.run("apt-get update") docker_env.device.run("apt-get install -y iptables") + bp(0) docker_env.device.run( "iptables -A OUTPUT -j DROP --destination docker.mender.io" ) @@ -240,21 +242,27 @@ def detect_shell_prompt(shell): # TCP RTO which means sometimes we need additional time to sleep. # this was exposed by the move to docker client in those tests, as the # network stack acts differently + bp(0) time.sleep(128) # Re-enable a good connection docker_env.device.run("iptables -D OUTPUT 1") time.sleep(30) + bp(0) # mender-connect should have "healed" now and be able to start a new shell - with docker_env.devconnect.get_websocket() as ws: - shell = proto_shell.ProtoShell(ws) - body = shell.startShell() - assert shell.protomsg.props["status"] == protomsg.PROP_STATUS_NORMAL - assert body == proto_shell.MSG_BODY_SHELL_STARTED - - detect_shell_prompt(shell) - is_shell_working(shell) + try: + with docker_env.devconnect.get_websocket() as ws: + shell = proto_shell.ProtoShell(ws) + body = shell.startShell() + bp(0) + assert shell.protomsg.props["status"] == protomsg.PROP_STATUS_NORMAL + assert body == proto_shell.MSG_BODY_SHELL_STARTED + + detect_shell_prompt(shell) + is_shell_working(shell) + except: + bp(0) @flaky(max_runs=3) def test_session_recording(self, docker_env): From 96065995eb5d5367463ca27f769559a5227af685 Mon Sep 17 00:00:00 2001 From: Peter Grzybowski Date: Wed, 20 May 2026 10:41:20 +0200 Subject: [PATCH 4/8] chore: debug Ticket: QA-1625 Signed-off-by: Peter Grzybowski --- tests/tests/test_mender_connect.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/tests/test_mender_connect.py b/tests/tests/test_mender_connect.py index b88675874..043a1b619 100644 --- a/tests/tests/test_mender_connect.py +++ b/tests/tests/test_mender_connect.py @@ -232,7 +232,7 @@ def detect_shell_prompt(shell): docker_env.device.run("apt-get update") docker_env.device.run("apt-get install -y iptables") - bp(0) + bp(1) docker_env.device.run( "iptables -A OUTPUT -j DROP --destination docker.mender.io" ) @@ -242,27 +242,27 @@ def detect_shell_prompt(shell): # TCP RTO which means sometimes we need additional time to sleep. # this was exposed by the move to docker client in those tests, as the # network stack acts differently - bp(0) + bp(2) time.sleep(128) # Re-enable a good connection docker_env.device.run("iptables -D OUTPUT 1") time.sleep(30) - bp(0) + bp(3) # mender-connect should have "healed" now and be able to start a new shell try: with docker_env.devconnect.get_websocket() as ws: shell = proto_shell.ProtoShell(ws) body = shell.startShell() - bp(0) + bp(4) assert shell.protomsg.props["status"] == protomsg.PROP_STATUS_NORMAL assert body == proto_shell.MSG_BODY_SHELL_STARTED detect_shell_prompt(shell) is_shell_working(shell) except: - bp(0) + bp(5) @flaky(max_runs=3) def test_session_recording(self, docker_env): From cb9ddd9ed793b1fc4b4fd3339bbc3fcea27c019a Mon Sep 17 00:00:00 2001 From: Peter Grzybowski Date: Wed, 20 May 2026 12:14:12 +0200 Subject: [PATCH 5/8] chore: debug Ticket: QA-1625 Signed-off-by: Peter Grzybowski --- tests/tests/test_mender_connect.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/tests/test_mender_connect.py b/tests/tests/test_mender_connect.py index 043a1b619..ec70f14aa 100644 --- a/tests/tests/test_mender_connect.py +++ b/tests/tests/test_mender_connect.py @@ -44,6 +44,9 @@ def bp(index): f="/tmp/bp"+str(index) + if not os.path.exists(f): + with open("/tmp/t.log", "a") as fh: + fh.write("waiting on "+f) while not os.path.exists(f): time.sleep(0.1) os.remove(f) From 6c102eeaae1f08dbd9120b8660f060c12378e0d7 Mon Sep 17 00:00:00 2001 From: Peter Grzybowski Date: Wed, 20 May 2026 13:29:59 +0200 Subject: [PATCH 6/8] chore: debug Ticket: QA-1625 Signed-off-by: Peter Grzybowski --- tests/tests/test_mender_connect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests/test_mender_connect.py b/tests/tests/test_mender_connect.py index ec70f14aa..fa5c0c975 100644 --- a/tests/tests/test_mender_connect.py +++ b/tests/tests/test_mender_connect.py @@ -46,7 +46,7 @@ def bp(index): f="/tmp/bp"+str(index) if not os.path.exists(f): with open("/tmp/t.log", "a") as fh: - fh.write("waiting on "+f) + fh.write("waiting on "+f+"\n") while not os.path.exists(f): time.sleep(0.1) os.remove(f) From 36bc3dd3b32cc83e0560bd2da221979999ad11b5 Mon Sep 17 00:00:00 2001 From: Peter Grzybowski Date: Wed, 20 May 2026 21:41:40 +0200 Subject: [PATCH 7/8] chore: debug Ticket: QA-1625 Signed-off-by: Peter Grzybowski --- tests/tests/test_mender_connect.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/tests/test_mender_connect.py b/tests/tests/test_mender_connect.py index fa5c0c975..5ac3714cc 100644 --- a/tests/tests/test_mender_connect.py +++ b/tests/tests/test_mender_connect.py @@ -40,12 +40,16 @@ from testutils.common import User, update_tenant from .common_connect import wait_for_connect +from datetime import datetime, timezone + container_factory = factory.get_factory() def bp(index): f="/tmp/bp"+str(index) if not os.path.exists(f): with open("/tmp/t.log", "a") as fh: + now_utc = datetime.now(timezone.utc) + now_utc.strftime("%a %b %d %H:%M:%S %Z %Y") fh.write("waiting on "+f+"\n") while not os.path.exists(f): time.sleep(0.1) @@ -245,13 +249,14 @@ def detect_shell_prompt(shell): # TCP RTO which means sometimes we need additional time to sleep. # this was exposed by the move to docker client in those tests, as the # network stack acts differently - bp(2) - time.sleep(128) + bp(2) # wait since Wed May 20 15:59:50 UTC 2026 -- dbg + # time.sleep(128) # Re-enable a good connection docker_env.device.run("iptables -D OUTPUT 1") - time.sleep(30) - bp(3) + # time.sleep(30) + bp(3) # -- Wed May 20 16:05:39 UTC 2026 dbg + # -- dbg Wed May 20 16:11:36 UTC 2026 # mender-connect should have "healed" now and be able to start a new shell try: From 5d1d072e5bdcbab4496170038ce82a2794455913 Mon Sep 17 00:00:00 2001 From: Peter Grzybowski Date: Thu, 21 May 2026 15:43:23 +0200 Subject: [PATCH 8/8] chore: debug i1 Ticket: QA-1625 Signed-off-by: Peter Grzybowski --- tests/tests/test_mender_connect.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/tests/test_mender_connect.py b/tests/tests/test_mender_connect.py index 5ac3714cc..8e21b9349 100644 --- a/tests/tests/test_mender_connect.py +++ b/tests/tests/test_mender_connect.py @@ -49,8 +49,7 @@ def bp(index): if not os.path.exists(f): with open("/tmp/t.log", "a") as fh: now_utc = datetime.now(timezone.utc) - now_utc.strftime("%a %b %d %H:%M:%S %Z %Y") - fh.write("waiting on "+f+"\n") + fh.write(now_utc.strftime("%a %b %d %H:%M:%S %Z %Y")+" waiting on "+f+"\n") while not os.path.exists(f): time.sleep(0.1) os.remove(f)