failed test should be reported rather than skipped and add tests

Author: Aniketsy

Lib/test/libregrtest/findtests.py

@@ -105,6 +105,10 @@ def list_cases(tests: TestTuple, *,
         print(count(len(skipped), "test"), "skipped:", file=stderr)
         printlist(skipped, file=stderr)
 
+class _ModuleLoadFailed(Exception):
+    """The test module failed to load; its test cases are unknown."""
+
+
 def collect_cases(tests: TestTuple, *,
                   match_tests: TestFilter | None = None,
                   test_dir: StrPath | None = None
@@ -124,16 +128,23 @@ def collect_cases(tests: TestTuple, *,
         except unittest.SkipTest:
             skipped.append(test_name)
             continue
+        except _ModuleLoadFailed:
+            # The module failed to load. Run it as a whole, so that the
+            # error is reported as in the normal mode.
+            result[test_name] = [test_name]
+            continue
         if cases:
             result[test_name] = cases
     return result, skipped
 
 def _collect_cases(suite: unittest.TestSuite, out: list[str]) -> None:
     for test in suite:
-        if isinstance(test, unittest.loader._FailedTest):  # type: ignore[attr-defined]
-            continue
         if isinstance(test, unittest.TestSuite):
             _collect_cases(test, out)
+        elif isinstance(test, unittest.loader._FailedTest):  # type: ignore[attr-defined]
+            # The test module failed to load. Its test cases are
+            # unknown: let the caller run the whole module.
+            raise _ModuleLoadFailed
         elif isinstance(test, unittest.TestCase):
             if match_test(test):
                 out.append(test.id())

Lib/test/libregrtest/main.py

@@ -558,13 +558,14 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
                 selected,
                 match_tests=self.match_tests,
                 test_dir=self.test_dir)
-            case_groups = tuple(
-                (module_name, tuple(cases))
-                for module_name, cases in cases_by_module.items()
-            )
-            if not case_groups:
-                self.log("No test cases found")
-                return 0
+            case_groups = []
+            for module_name in selected:
+                cases = cases_by_module.get(module_name)
+                if cases:
+                    case_groups.append((module_name, tuple(cases)))
+                else:
+                    case_groups.append((module_name, (module_name,)))
+            case_groups = tuple(case_groups)
             case_ids = tuple(
                 case_id for _, cases in case_groups for case_id in cases
             )

Lib/test/libregrtest/run_workers.py

@@ -274,9 +274,7 @@ def create_worker_runtests(self, test_name: TestName,
                                ) -> WorkerRunTests:
         kwargs: dict[str, Any] = {}
 
-        if module_name is not None:
-            # single_process_per_case mode: run a single test case
-            # (test_name is a case ID) inside its module.
+        if module_name is not None and test_name != module_name:
             tests = (module_name,)
             kwargs['match_tests'] = [(test_name, True)]
         else:
@@ -426,7 +424,7 @@ def run(self) -> None:
                     finally:
                         self.test_name = _NOT_RUNNING
                     mp_result.result.duration = time.monotonic() - self.start_time
-                    if single_process_per_case:
+                    if single_process_per_case and test_name != module_name:
                         # Report the test case, not the test module
                         mp_result.result.test_name = test_name
                     self.output.put((False, mp_result))

Lib/test/test_regrtest.py

@@ -2427,6 +2427,26 @@ def test_fail(self):
         self.assertIn(f'{testname}.Tests.test_pass passed', output)
         self.check_line(output, '1 test OK.', regex=False)
 
+    def test_single_process_per_case_import_error(self):
+        testname = self.create_test(code='raise ImportError("boom")')
+
+        output = self.run_tests('--single-process-per-case', testname,
+                                exitcode=EXITCODE_BAD_TEST)
+        self.check_executed_tests(output, [testname], failed=[testname],
+                                  stats=0)
+        self.assertNotIn('_FailedTest', output)
+
+    def test_single_process_per_case_skipped_module(self):
+        code = textwrap.dedent("""
+            import unittest
+            raise unittest.SkipTest("nope")
+        """)
+        testname = self.create_test(code=code)
+
+        output = self.run_tests('--single-process-per-case', testname)
+        self.check_executed_tests(output, [testname],
+                                  skipped=[testname], stats=0)
+
 
     def test_verbose3(self):
         code = textwrap.dedent(r"""