chore: send mender_connect ent tests to other runner via pytest-order

.gitlab-ci-full-integration-template.yml

@@ -49,8 +49,11 @@ test:integration:$CI_NODE_INDEX:
       fi
     # Increase system limits to make sure the tests are not limited while
     # running with high parallelism on a single VM
+    - sysctl net.ipv4.tcp_retries1
+    - sysctl net.ipv4.tcp_retries2
     - sysctl -w fs.inotify.max_user_instances=1024
     - sysctl -w fs.file-max=600000
+    - sysctl -w net.ipv4.tcp_retries2=3
     - ulimit -n 524288
 
   script:

tests/requirements-python/python-requirements.in

@@ -9,6 +9,7 @@ pytest-html==4.2.0
 pytest-metadata==3.1.1
 pytest-timeout==2.4.0
 pytest-xdist==3.8.0
+pytest-order==1.1.0
 redo==3.0.0
 requests==2.33.1
 urllib3==2.6.3

tests/requirements-python/python-requirements.txt

@@ -69,6 +69,8 @@ pytest-timeout==2.4.0
     # via -r python-requirements.in
 pytest-xdist==3.8.0
     # via -r python-requirements.in
+pytest-order==1.1.0
+    # via -r python-requirements.in
 redo==3.0.0
     # via -r python-requirements.in
 requests==2.33.1

tests/run.sh

@@ -145,6 +145,7 @@ if test ${CI_NODE_TOTAL:-1} -gt 1; then
 fi
 python3 -m pytest \
     $EXTRA_TEST_ARGS \
+    --dist=loadfile \
     --verbose \
     --junitxml=results.xml \
     $HTML_REPORT \

tests/tests/test_mender_connect.py → tests/tests/test_enterprise_mender_connect.py

@@ -188,6 +188,91 @@ def test_bogus_shell_message(self, docker_env):
             assert prot.protoType == proto_shell.PROTO_TYPE_SHELL
             assert prot.typ == "bogusmessage"
 
+    @pytest.mark.order(-1)
+    @flaky(max_runs=3)
+    def test_in_poor_network_environment(self, docker_env):
+        self.assert_env(docker_env)
+
+        receive_timeout_s = 16
+
+        def is_shell_working(shell):
+            # Test if a simple command works.
+            shell.sendInput("ls /\n".encode())
+            output = shell.recvOutput(receive_timeout_s)
+            assert shell.protomsg.props["status"] == protomsg.PROP_STATUS_NORMAL
+            output = output.decode()
+            assert "usr" in output
+            assert "etc" in output
+
+        def detect_shell_prompt(shell):
+            # Drain any initial output from the prompt. It should end in either "# "
+            # (root) or "$ " (user).
+            output = shell.recvOutput(receive_timeout_s)
+            assert shell.protomsg.props["status"] == protomsg.PROP_STATUS_NORMAL
+            assert output[-2:].decode() in [
+                "# ",
+                "$ ",
+            ], "Could not detect shell prompt."
+
+        with docker_env.devconnect.get_websocket() as ws:
+            shell = proto_shell.ProtoShell(ws)
+            body = shell.startShell()
+            assert shell.protomsg.props["status"] == protomsg.PROP_STATUS_NORMAL
+            assert body == proto_shell.MSG_BODY_SHELL_STARTED
+
+            detect_shell_prompt(shell)
+            is_shell_working(shell)
+
+        docker_env.device.run("apt-get update")
+        docker_env.device.run("apt-get install -y iptables curl")
+        docker_env.device.run(
+            "iptables -A OUTPUT -j DROP --destination docker.mender.io"
+        )
+
+        # Plenty of time for the session to mess up
+        # see also QA-1591: the DROP will not cause ICMP response so we rely on the
+        # TCP RTO which means sometimes we need additional time to sleep.
+        # this was exposed by the move to docker client in those tests, as the
+        # network stack acts differently
+        time.sleep(128)
+
+        # Re-enable a good connection
+        docker_env.device.run("iptables -D OUTPUT 1")
+        docker_env.device.run("iptables -F OUTPUT")
+        docker_env._docker_compose_cmd("exec mender-client iptables -F OUTPUT")
+        connectivity_reestablished = False
+        sleep_time_s = 8
+        iteration = 0
+        max_iteration = 64
+        while not connectivity_reestablished:
+            rc = docker_env._docker_compose_cmd(
+                'exec mender-client curl -s -o /dev/null -w "%{http_code}" -k --connect-timeout 240 --keepalive-time 30 --max-time 300 --retry-delay 8 https://docker.mender.io'
+            )
+            if rc.startswith("20"):
+                connectivity_reestablished = True
+                break
+            if rc.startswith("30"):
+                connectivity_reestablished = True
+                break
+            time.sleep(sleep_time_s)
+            iteration = iteration + 1
+            if iteration > max_iteration:
+                break
+        assert (
+            connectivity_reestablished
+        ), "connectivity was not reestablished; this is a test env failure"
+        time.sleep(128)
+
+        # mender-connect should have "healed" now and be able to start a new shell
+        with docker_env.devconnect.get_websocket() as ws:
+            shell = proto_shell.ProtoShell(ws)
+            body = shell.startShell()
+            assert shell.protomsg.props["status"] == protomsg.PROP_STATUS_NORMAL
+            assert body == proto_shell.MSG_BODY_SHELL_STARTED
+
+            detect_shell_prompt(shell)
+            is_shell_working(shell)
+
     @flaky(max_runs=3)
     def test_session_recording(self, docker_env):
         self.assert_env(docker_env)
@@ -317,26 +402,6 @@ def test_bogus_proto_message(self, docker_env):
             assert isinstance(body.get("err"), str) and len(body.get("err")) > 0
 
 
-class TestRemoteTerminalOpenSource(
-    _TestRemoteTerminalBase, _TestRemoteTerminalBaseBogusProtoMessage
-):
-    @pytest.fixture(scope="function")
-    def docker_env(self, standard_setup_one_docker_client_bootstrapped):
-        env = standard_setup_one_docker_client_bootstrapped
-        auth = Authentication()
-        env.devconnect = DeviceConnect(auth, DeviceAuthV2(auth))
-        yield env
-
-    @pytest.fixture(scope="function")
-    def docker_env_flaky_test(
-        self, request, standard_setup_one_docker_client_bootstrapped
-    ):
-        env = standard_setup_one_docker_client_bootstrapped
-        auth = Authentication()
-        env.devconnect = DeviceConnect(auth, DeviceAuthV2(auth))
-        yield env
-
-
 def connected_device(env):
     uuidv4 = str(uuid.uuid4())
     tname = "test.mender.io-{}".format(uuidv4)
@@ -403,63 +468,3 @@ def docker_env_flaky_test(self, enterprise_one_docker_client_bootstrapped):
         env.devconnect = devconn
 
         yield env
-
-    def test_in_poor_network_environment(self, docker_env):
-        self.assert_env(docker_env)
-
-        receive_timeout_s = 16
-
-        def is_shell_working(shell):
-            # Test if a simple command works.
-            shell.sendInput("ls /\n".encode())
-            output = shell.recvOutput(receive_timeout_s)
-            assert shell.protomsg.props["status"] == protomsg.PROP_STATUS_NORMAL
-            output = output.decode()
-            assert "usr" in output
-            assert "etc" in output
-
-        def detect_shell_prompt(shell):
-            # Drain any initial output from the prompt. It should end in either "# "
-            # (root) or "$ " (user).
-            output = shell.recvOutput(receive_timeout_s)
-            assert shell.protomsg.props["status"] == protomsg.PROP_STATUS_NORMAL
-            assert output[-2:].decode() in [
-                "# ",
-                "$ ",
-            ], "Could not detect shell prompt."
-
-        with docker_env.devconnect.get_websocket() as ws:
-            shell = proto_shell.ProtoShell(ws)
-            body = shell.startShell()
-            assert shell.protomsg.props["status"] == protomsg.PROP_STATUS_NORMAL
-            assert body == proto_shell.MSG_BODY_SHELL_STARTED
-
-            detect_shell_prompt(shell)
-            is_shell_working(shell)
-
-        docker_env.device.run("apt-get update")
-        docker_env.device.run("apt-get install -y iptables")
-        docker_env.device.run(
-            "iptables -A OUTPUT -j DROP --destination docker.mender.io"
-        )
-
-        # Plenty of time for the session to mess up
-        # see also QA-1591: the DROP will not cause ICMP response so we rely on the
-        # TCP RTO which means sometimes we need additional time to sleep.
-        # this was exposed by the move to docker client in those tests, as the
-        # network stack acts differently
-        time.sleep(128)
-
-        # Re-enable a good connection
-        docker_env.device.run("iptables -D OUTPUT 1")
-        time.sleep(128)
-
-        # mender-connect should have "healed" now and be able to start a new shell
-        with docker_env.devconnect.get_websocket() as ws:
-            shell = proto_shell.ProtoShell(ws)
-            body = shell.startShell()
-            assert shell.protomsg.props["status"] == protomsg.PROP_STATUS_NORMAL
-            assert body == proto_shell.MSG_BODY_SHELL_STARTED
-
-            detect_shell_prompt(shell)
-            is_shell_working(shell)