Merge pull request #151 from Tencent/bugfix/load-testsuite

Bugfix/load testsuite

testbase/loader.py

@@ -112,9 +112,11 @@ def load(self, testname):
                         obj, data_key, exclude_data_keys, parameters
                     )
                 elif issubclass(obj, TestSuite):
-                    testcases += self._load_from_testsuite(
-                        obj, data_key, exclude_data_keys, parameters
-                    )
+                    if not self._filter_testcase(obj):
+                        tests = self._load_from_testsuite(
+                            obj, data_key, exclude_data_keys, parameters
+                        )
+                        testcases.append(obj(tests))  # append testsuite to testcase list
 
         # 过滤掉重复的用例
         testcase_dict = OrderedDict()
@@ -192,13 +194,24 @@ def _is_testsuite_class(self, obj):
 
         :returns bool - 是否为用例套类
         """
-        return isinstance(obj, type) and issubclass(obj, TestSuite)
+        return (
+            isinstance(obj, type)
+            and issubclass(obj, TestSuite)
+            and obj is not TestSuite
+        )
 
     def _walk_package_error(self, modulename):
         """walk_packages错误回调"""
         self._module_errs[modulename] = traceback.format_exc()
 
-    def _load_from_package(self, pkg, data_key=None, exclude_data_key=None, attrs=None):
+    def _load_from_package(
+        self,
+        pkg,
+        data_key=None,
+        exclude_data_key=None,
+        attrs=None,
+        ignore_testsuite=False,
+    ):
         """从一个python包加载测试用例
 
         :param pkg: Python包
@@ -218,12 +231,20 @@ def _load_from_package(self, pkg, data_key=None, exclude_data_key=None, attrs=No
                     data_key,
                     exclude_data_key=exclude_data_key,
                     attrs=None,
+                    ignore_testsuite=ignore_testsuite,
                 )
             except Exception:  # pylint: disable=broad-except
                 self._module_errs[modulename] = traceback.format_exc()
         return tests
 
-    def _load_from_module(self, mod, data_key=None, exclude_data_key=None, attrs=None):
+    def _load_from_module(
+        self,
+        mod,
+        data_key=None,
+        exclude_data_key=None,
+        attrs=None,
+        ignore_testsuite=False,
+    ):
         """从一个python模块加载测试用例
 
         :param mod: Python模块
@@ -237,6 +258,13 @@ def _load_from_module(self, mod, data_key=None, exclude_data_key=None, attrs=Non
                 tests += self._load_from_class(
                     obj, data_key, exclude_data_key=exclude_data_key, attrs=attrs
                 )
+            elif self._is_testsuite_class(obj) and not ignore_testsuite:
+                if not self._filter_testcase(obj):
+                    testcases = self._load_from_testsuite(
+                        obj, data_key, exclude_data_key=exclude_data_key, attrs=attrs
+                    )
+                    tests.append(obj(testcases))
+
         if hasattr(mod, "__qtaf_seq_tests__"):  # 测试用例需要顺序执行
             seqdef = mod.__qtaf_seq_tests__
             if not isinstance(seqdef, list):
@@ -274,18 +302,36 @@ def _load_from_testsuite(
             if not isinstance(test, type):
                 if hasattr(test, "__path__"):
                     tests += self._load_from_package(
-                        test, data_key, exclude_data_key=exclude_data_key, attrs=attrs
+                        test,
+                        data_key,
+                        exclude_data_key=exclude_data_key,
+                        attrs=attrs,
+                        ignore_testsuite=True,
                     )
                 else:
                     tests += self._load_from_module(
-                        test, data_key, exclude_data_key=exclude_data_key, attrs=attrs
+                        test,
+                        data_key,
+                        exclude_data_key=exclude_data_key,
+                        attrs=attrs,
+                        ignore_testsuite=True,
                     )
             else:
                 tests += self._load_from_class(
                     test, data_key, exclude_data_key=exclude_data_key, attrs=attrs
                 )
 
-        return [cls([it for it in tests if not cls.filter(it)])]
+        return [it for it in tests if not cls.filter(it)]
+
+    def _filter_testcase(self, test):
+        if not self._filter:
+            return False
+        filter_reason = self._filter(test)
+        if filter_reason:
+            self._filtered_tests[test] = filter_reason
+            return True
+        else:
+            return False
 
     def _load_from_class(self, cls, data_key=None, exclude_data_key=None, attrs=None):
         """加载用例类
@@ -312,16 +358,7 @@ def _load_from_class(self, cls, data_key=None, exclude_data_key=None, attrs=None
         else:
             tests = [cls(attrs=attrs)]
 
-        if self._filter:
-            final_tests = []
-            for it in tests:
-                filter_reason = self._filter(it)
-                if filter_reason:
-                    self._filtered_tests[it] = filter_reason
-                else:
-                    final_tests.append(it)
-            tests = final_tests
-        return tests
+        return [it for it in tests if not self._filter_testcase(it)]
 
 
 class TestDataLoader(object):

testbase/management.py

@@ -34,6 +34,7 @@
 from testbase.dist import DistGenerator, VirtuelEnv
 from testbase.loader import TestLoader
 from testbase.testcase import TestCasePriority, TestCaseStatus, TestCase
+from testbase.testsuite import TestSuite
 from testbase.runner import TestCaseSettings
 from testbase.report import test_list_types
 from testbase.types import runner_types, report_types, resmgr_backend_types
@@ -526,7 +527,7 @@ def execute(self, args):
         if not args.tests:
             logger.info("no test set specified")
             return 1
-        shows = args.shows or ["filtered", "error", "normal"]
+        shows = args.shows or ["filtered", "error", "normal", "testsuite"]
         priorities = args.priorities or [
             TestCase.EnumPriority.Normal,
             TestCase.EnumPriority.High,
@@ -545,6 +546,11 @@ def execute(self, args):
 
         loader = TestLoader(test_conf.filter)
         tests = loader.load(test_conf.names)
+        testsuites = []
+        for test in tests[:]:
+            if isinstance(test, TestSuite):
+                testsuites.append(test)
+                tests.remove(test)
 
         test_list_output = test_list_types[args.output_type](args.output_file)
 
@@ -561,6 +567,10 @@ def execute(self, args):
             error_tests = loader.get_last_errors().items()
             sorted_error_tests = sorted(error_tests, key=lambda x: x[0])
             test_list_output.output_error_tests(sorted_error_tests)
+
+        if "testsuite" in shows:
+            test_list_output.output_testsuites(testsuites)
+
         test_list_output.end_output()
 
 

testbase/report.py

@@ -1047,6 +1047,9 @@ def output_filtered_tests(self, filtered_tests):
     def output_error_tests(self, error_tests):
         raise NotImplementedError
 
+    def output_testsuites(self, testsuites):
+        raise NotImplementedError
+
     def end_output(self):
         if self._close_fd:
             self._fd.close()
@@ -1079,6 +1082,14 @@ def output_error_tests(self, error_tests):
             test_info = 'cannot load test "%s"' % test_name + ", error:\n" + error
             self._output_func(test_info)
 
+    def output_testsuites(self, testsuites):
+        self._output_func("\n======================")
+        self._output_func("%s testsuites:" % len(testsuites))
+        self._output_func("======================")
+        for testsuite in testsuites:
+            test_info = self.stream_format_test(testsuite)
+            self._output_func(test_info)
+
     def stream_format_test(self, test):
         desc = "%s/%s" % (test.priority, test.status)
         test_info = "%-12s " % desc

testbase/testsuite.py

@@ -223,7 +223,7 @@ def sequential_run(
         :return TestResult/TestResultCollection - 测试结果
         """
         passed = True
-        results = []
+        results = [testresult]
         for it in testsuite:
             case_result = self._run_test(testsuite, it, testresult_factory, testresult)
             passed &= case_result.passed
@@ -276,7 +276,7 @@ def parallel_run(self, testsuite, testresult_factory, testresult, concurrency):
         :return TestResult/TestResultCollection - 测试结果
         """
         tests_queue = collections.deque([it for it in testsuite])
-        results = []
+        results = [testresult]
         threads = []
         lock = threading.Lock()
         for _ in range(concurrency):
@@ -318,7 +318,7 @@ def run(self, testsuite, testresult_factory):
         testresult.begin_test(testsuite)
         testresult.begin_step("pre_test")
         testsuite.init_test(testresult)
-        result = TestResultCollection([], False)
+        result = TestResultCollection([testresult], False)
         try:
             testsuite.pre_test()
         except:
@@ -369,6 +369,10 @@ class EnumExecMode(object):
         Parallel = "parallel"
         Sequential = "sequential"
 
+    owner = None
+    priority = None
+    status = None
+    timeout = None
     testcases = []
     testcase_filter = {}
     exec_mode = EnumExecMode.Sequential
@@ -382,8 +386,7 @@ def __init__(self, testcases):
             concurrency=self.concurrency,
         )
         self.__testcases = testcases
-        self.__testresult = None
-        self.__testresults = None
+        self.__testresults = []
         self.__current_stage = ""
 
     def __iter__(self):
@@ -415,15 +418,19 @@ def test_result(self):
 
         :return: TestResult
         """
-        return self.__testresult
+        if self.__testresults:
+            return self.__testresults[0]
+        return None
 
     @property
     def test_results(self):
         """测试用例结果
 
         :return: TestResultCollection
         """
-        return self.__testresults
+        if len(self.__testresults) > 1:
+            return self.__testresults[1:]
+        return []
 
     @property
     def current_stage(self):
@@ -461,6 +468,7 @@ def filter(cls, testcase):
         tag_list = cls.testcase_filter.get("tags", [])
         if (
             tag_list
+            and hasattr(testcase, "tags")
             and isinstance(tag_list, list)
             and set(tag_list).isdisjoint(testcase.tags)
         ):
@@ -510,7 +518,7 @@ def init_test(self, testresult):
         :param testresult: 测试结果
         :type testresult: TestResult
         """
-        self.__testresult = testresult
+        self.__testresults.append(testresult)
 
     def pre_test(self):
         pass

tests/test_testbase/test_testsuite.py

@@ -76,6 +76,7 @@ def test_load_from_testsuite(self):
         testsuite_class.testcase_filter = {}
         tests = list(testloader.load(testsuite)[0])
         self.assertEqual(len(tests), 1)
+        self.assertEqual(tests[0].__class__.__name__, "HelloTest")
 
     def test_filter(self):
         testloader = TestLoader()