Feature/stop on failure

testbase/management.py

@@ -24,6 +24,7 @@
 import traceback
 import sys
 import shlex
+from xmlrpc.client import Boolean, boolean
 import six
 
 from testbase import logger
@@ -186,11 +187,11 @@ class RunTest(Command):
 
     parser.add_argument("--share-data", help="share data", default="")
     parser.add_argument("--global-parameters", help="global parameters", default="")
-
+    parser.add_argument("--stop-on-failure", help="when the testcase execute fail, the test task will stop", default=False)
     parser.add_argument("--config-file", help="runtime config file path")
 
     def run_args_parser(self, runner_args):
-        """兼容参数传入concurrency=5,retries=1，支持subprocess shell=False"""
+        """兼容参数传入concurrency=5,retries=1, 支持subprocess shell=False"""
         regex_c = re.compile(r'(\w+=\w+)+')
         regex = regex_c.search(runner_args)
         if regex:
@@ -246,6 +247,9 @@ def execute(self, args):
 
         if args.global_parameters and isinstance(args.global_parameters, six.string_types):
             args.global_parameters = json.loads(args.global_parameters)
+
+        # if args.stop_on_failure and isinstance(args.stop_on_failure, six.string_types):
+        #     args.stop_on_failure = json.loads(args.stop_on_failure)
 
         test_conf = TestCaseSettings(names=args.tests,
                                     excluded_names=args.excluded_names,

testbase/report.py

@@ -14,7 +14,6 @@
 #
 """测试报告
 """
-
 import sys
 import socket
 import os
@@ -109,6 +108,17 @@ def log_filtered_test(self, loader, testcase, reason):
         """
         pass
 
+    def log_ignored_test(self, testcase, reason):
+        """
+        记录一个忽略的测试用例
+
+        :param  testcase:   测试用例
+        :type   testcase:   TestCase实例
+        :param  reason:     忽略原因
+        :type   reason:     str
+        """
+        pass
+
     def log_load_error(self, loader, name, error):
         """记录一个加载失败的用例或用例集
 
@@ -513,6 +523,19 @@ def log_filtered_test(self, loader, testcase, reason):
         self._write(
             "filtered test case: %s (reason: %s)\n" % (testcase.test_name, reason)
         )
+
+    def log_ignored_test(self, testcase, reason):
+        """
+        记录一个被忽略的测试用例
+        :param  testcase:   测试用例
+        :type   testcase:   TestCase
+        :param  reason:     忽略原因
+        :type   reason:     str
+        """
+
+        self._write(
+            "ignored test case: %s (rease %s)\n" % (testcase.test_name, reason)
+        )
 
     def log_load_error(self, loader, name, error):
         """记录一个加载失败的用例或用例集
@@ -693,6 +716,24 @@ def log_filtered_test(self, loader, testcase, reason):
         doc2 = dom.parseString(nodestr)
         filterNode = doc2.childNodes[0]
         self._runrstnode.appendChild(filterNode)
+
+    def log_ignored_test(self, testcase, reason):
+        """
+        记录一个被忽略的测试用例
+        :param  testcase:   测试用例
+        :type   testcase:   TestCase
+        :param  reason:     忽略原因
+        :type   reason:     str
+        """
+
+        nodestr = """<IgnoreTest name="%s" reason="%s"><IgnoreTest>
+        """ % (
+            smart_text(saxutils.escape(testcase.test_name)),
+            smart_text(saxutils.escape(reason)),
+        )
+        doc2 = dom.parseString(nodestr)
+        ignoreNode = doc2.childNodes[0]
+        self._runrstnode.appendChild(ignoreNode)
 
     def log_load_error(self, loader, name, error):
         """记录一个加载失败的用例或用例集
@@ -764,6 +805,7 @@ def __init__(self, title="调试测试"):
         super(JSONTestReportBase, self).__init__()
         self._logs = []
         self._filtered_tests = []
+        self._ignored_tests = []
         self._load_errors = []
         self._passed_tests = {}
         self._failed_tests = {}
@@ -869,6 +911,16 @@ def log_filtered_test(self, loader, testcase, reason):
         :type reason: str
         """
         self._filtered_tests.append({"name": testcase.test_name, "reason": reason})
+
+    def log_ignored_test(self, testcase, reason):
+        """
+        记录一个被忽略的测试用例
+        :param  testcase:   测试用例
+        :type   testcase:   TestCase
+        :param  reason:     忽略原因
+        :type   reason:     str
+        """
+        self._ignored_tests.append({"name": testcase.test_name, "reason": reason})
 
     def log_load_error(self, loader, name, error):
         """记录一个加载失败的用例或用例集

testbase/runner.py

@@ -48,7 +48,7 @@
 
 from testbase.loader import TestLoader
 from testbase import serialization
-from testbase.testcase import TestCase, TestCaseRunner, TestSuite
+from testbase.testcase import TestCase, TestCaseRunner, TestCaseType, TestSuite
 from testbase.report import TestReportBase
 from testbase.testresult import TestResultCollection
 from testbase.resource import TestResourceManager, LocalResourceManagerBackend
@@ -63,7 +63,7 @@ class TestCaseSettings(object):
     '''
 
     def __init__(self, names=None, excluded_names=None, priorities=None, status=None, owners=None,
-                tags=None, excluded_tags=None, share_data={}, global_parameters={}):
+                tags=None, excluded_tags=None, share_data={}, global_parameters={}, stop_on_failure=False):
         '''构造函数
 
         :param names: 测试用例名
@@ -80,6 +80,8 @@ def __init__(self, names=None, excluded_names=None, priorities=None, status=None
         :type tags: list
         :param excluded_tags: 指定标签排除用例
         :type tags: list
+        :param stop_on_failure: 失败停止用例执行
+        :type stop_on_failure: bool
         '''
         if names is None:
             self.names = []
@@ -135,6 +137,7 @@ def __init__(self, names=None, excluded_names=None, priorities=None, status=None
         self.tags = set(tags) if tags else None
         self.excluded_tags = set(excluded_tags) if excluded_tags else None
         self.share_data = share_data
+        self.stop_on_failure = stop_on_failure
 
     def _generate_testcase_names(self, testcase):
         if testcase.endswith(".py"):
@@ -245,6 +248,7 @@ def load(self, target):
             tests = loader.load(target.names)
             self.__report.log_loaded_tests(loader, tests)
             filtered_tests = loader.get_filtered_tests_with_reason().items()
+            self._stop_on_failure = target.stop_on_failure
             for test, reason in filtered_tests:
                 self.__report.log_filtered_test(loader, test, reason)
             for testname, error in loader.get_last_errors().items():
@@ -404,6 +408,10 @@ def run_all_tests(self, tests):
             test = tests_queue.popleft()
             passed = self.run_test(test)
             if not passed:
+                if self._stop_on_failure:
+                    reason = "The execution of the previous test case failed, setting current testcase ignored."
+                    for case in tests_queue:
+                        self.__report.log_ignored_test(case, reason)
                 tests_retry_dict.setdefault(test, 0)
                 if tests_retry_dict[test] < self._retries:
                     tests_retry_dict[test] += 1
@@ -494,6 +502,17 @@ def log_filtered_test(self, loader, testcase, reason):
         '''
         with self._lock:
             return self._report.log_filtered_test(loader, testcase, reason)
+
+    def log_ignored_test(self, testcase, reason):
+        """
+        记录一个被忽略的测试用例
+        :param  testcase:   测试用例
+        :type   testcase:   TestCase
+        :param  reason:     忽略原因
+        :type   reason:     str
+        """
+        with self._lock:
+            return self._report.log_ignored_test(testcase, reason)
 
     def log_load_error(self, loader, name, error):
         '''记录一个加载失败的用例或用例集
@@ -586,10 +605,16 @@ def _run_test_from_queue(self, tests_queue, tests_retry_dict):
             passed = self.run_test(test)
             with self._lock:
                 if not passed:
-                    tests_retry_dict.setdefault(test, 0)
-                    if tests_retry_dict[test] < self._retries:
-                        tests_retry_dict[test] += 1
-                        tests_queue.append(test)
+                    if self._stop_on_failure:
+                        reason = "The execution of the previous test case failed, setting current testcase ignored."
+                        for case in tests_queue:
+                            self.__report.log_ignored_test(case, reason)
+                        tests_queue.clear()
+                    else:
+                        tests_retry_dict.setdefault(test, 0)
+                        if tests_retry_dict[test] < self._retries:
+                            tests_retry_dict[test] += 1
+                            tests_queue.append(test)
 
     @classmethod
     def get_parser(cls):
@@ -1120,10 +1145,16 @@ def run_all_tests(self, tests):
                 test = serialization.loads(msg[2])
                 passed = msg[3]
                 if not passed:
-                    tests_retry_dict.setdefault(test.test_name, 0)
-                    if tests_retry_dict[test.test_name] < self._retries:
-                        tests_retry_dict[test.test_name] += 1
-                        tests_queue.append(test)
+                    if self._stop_on_failure:
+                        for case in tests_queue:
+                            reason = "The execution of the previous test case failed, setting current testcase ignored."
+                            self.__report.log_ignored_test(case, reason)
+                        tests_queue.clear()
+                    else:
+                        tests_retry_dict.setdefault(test.test_name, 0)
+                        if tests_retry_dict[test.test_name] < self._retries:
+                            tests_retry_dict[test.test_name] += 1
+                            tests_queue.append(test)
 
                 if len(tests_queue) > 0:
                     worker.run_testcase(tests_queue.popleft())

testbase/testcase.py

@@ -164,7 +164,7 @@ class EnumStatus(object):
                                                      TestCaseStatus.Implement,
                                                      TestCaseStatus.Review,
                                                      TestCaseStatus.Ready,
-                                                     TestCaseStatus.Suspend)
+                                                     TestCaseStatus.Suspend,)
 
     class EnumPriority(object):
         '''测试用例优先级枚举类

tests/sampletest/stoponfailuretest.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+'''
+参数测试用例
+'''
+
+import testbase
+import time
+
+
+class FirstFailureTest(testbase.TestCase):
+    """
+    参数测试用例
+    """
+    owner = "foo"
+    status = testbase.TestCase.EnumStatus.Ready
+    priority = testbase.TestCase.EnumPriority.Normal
+    timeout = 1
+
+    def run_test(self):
+        self.assertEqual("1", "2")
+
+class SecondFailureTest(testbase.TestCase):
+    """
+    参数测试用例
+    """
+    owner = "foo"
+    status = testbase.TestCase.EnumStatus.Ready
+    priority = testbase.TestCase.EnumPriority.Normal
+    timeout = 1
+
+
+    def run_test(self):
+        self.assertEqual("hello", "world")
+

tests/test_testbase/test_runner.py

@@ -86,6 +86,9 @@ def log_loaded_tests(self, loader, testcases):
 
     def log_filtered_test(self, loader, testcase, reason):
         self.logs.append(["log_filtered_test", loader, testcase, reason])
+
+    def log_ignored_test(self, testcase, reason):
+        self.logs.append(["log_ignored_test", testcase, reason])
 
     def log_load_error(self, loader, name, error):
         self.logs.append(["log_load_error", loader, name, error])
@@ -314,7 +317,23 @@ def test_execute_type(self):
             if runner_type == runner.MultiProcessTestRunner:
                 sys.modules["__main__"] = old_main
                 sys.modules["__main__"].__file__ = old_main_file
+
+    def test_stop_on_failure(self):
+        runner_types = [runner.TestRunner, runner.ThreadingTestRunner, runner.MultiProcessTestRunner]
 
+        for runner_type in runner_types:
+            if runner_type == runner.MultiProcessTestRunner:
+                import sys
+                old_main = sys.modules["__main__"]
+                old_main_file = sys.modules["__main__"].__file__
+                sys.modules["__main__"] = sys.modules[__name__]
+                sys.modules["__main__"].__file__ = sys.modules[__name__].__file__
+
+            report = TestReport()
+            r = runner_type(report, execute_type="random")
+            r.run(runner.TestCaseSettings(["tests.sampletest.stoponfailuretest.FirstFailureTest", "tests.sampletest.stoponfailuretest.SecondFailureTest"]))
+            for it in report.logs:
+                print(it[0])
 
 if __name__ == "__main__":
     unittest.main()