bpo-36719: Fix regrtest re-run (GH-12964)

vstinner · web-flow · commit 837acc1957d8 · 2019-04-26T09:56:37.000+02:00
Properly handle a test which fail but then pass.

Add test_rerun_success() unit test.
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
@@ -103,17 +103,18 @@ def get_executed(self):
                 | set(self.resource_denieds) | set(self.environment_changed)
                 | set(self.run_no_tests))
 
-    def accumulate_result(self, result):
+    def accumulate_result(self, result, rerun=False):
         test_name = result.test_name
         ok = result.result
 
-        if ok not in (CHILD_ERROR, INTERRUPTED):
+        if ok not in (CHILD_ERROR, INTERRUPTED) and not rerun:
             self.test_times.append((result.test_time, test_name))
 
         if ok == PASSED:
             self.good.append(test_name)
         elif ok in (FAILED, CHILD_ERROR):
-            self.bad.append(test_name)
+            if not rerun:
+                self.bad.append(test_name)
         elif ok == ENV_CHANGED:
             self.environment_changed.append(test_name)
         elif ok == SKIPPED:
@@ -123,9 +124,14 @@ def accumulate_result(self, result):
             self.resource_denieds.append(test_name)
         elif ok == TEST_DID_NOT_RUN:
             self.run_no_tests.append(test_name)
-        elif ok != INTERRUPTED:
+        elif ok == INTERRUPTED:
+            self.interrupted = True
+        else:
             raise ValueError("invalid test result: %r" % ok)
 
+        if rerun and ok not in {FAILED, CHILD_ERROR, INTERRUPTED}:
+            self.bad.remove(test_name)
+
         xml_data = result.xml_data
         if xml_data:
             import xml.etree.ElementTree as ET
@@ -287,13 +293,11 @@ def rerun_failed_tests(self):
         for test_name in self.rerun:
             print(f"Re-running {test_name} in verbose mode", flush=True)
             self.ns.verbose = True
-            ok = runtest(self.ns, test_name)
+            result = runtest(self.ns, test_name)
 
-            if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}:
-                self.bad.remove(test_name)
+            self.accumulate_result(result, rerun=True)
 
-            if ok.result == INTERRUPTED:
-                self.interrupted = True
+            if result.result == INTERRUPTED:
                 break
 
         if self.bad:
@@ -392,7 +396,6 @@ def run_tests_sequential(self):
                 self.accumulate_result(result)
 
             if result.result == INTERRUPTED:
-                self.interrupted = True
                 break
 
             previous_test = format_test_result(result)
diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py
@@ -255,9 +255,6 @@ def _process_result(self, item):
         if mp_result.stderr and not self.ns.pgo:
             print(mp_result.stderr, file=sys.stderr, flush=True)
 
-        if mp_result.result.result == INTERRUPTED:
-            self.regrtest.interrupted = True
-
         if must_stop(mp_result.result):
             return True
 
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
@@ -484,7 +484,7 @@ def list_regex(line_format, tests):
             result.append('SUCCESS')
         result = ', '.join(result)
         if rerun:
-            self.check_line(output, 'Tests result: %s' % result)
+            self.check_line(output, 'Tests result: FAILURE')
             result = 'FAILURE then %s' % result
 
         self.check_line(output, 'Tests result: %s' % result)
@@ -989,6 +989,7 @@ def test_env_changed(self):
                                   fail_env_changed=True)
 
     def test_rerun_fail(self):
+        # FAILURE then FAILURE
         code = textwrap.dedent("""
             import unittest
 
@@ -1003,6 +1004,26 @@ def test_bug(self):
         self.check_executed_tests(output, [testname],
                                   failed=testname, rerun=testname)
 
+    def test_rerun_success(self):
+        # FAILURE then SUCCESS
+        code = textwrap.dedent("""
+            import builtins
+            import unittest
+
+            class Tests(unittest.TestCase):
+                failed = False
+
+                def test_fail_once(self):
+                    if not hasattr(builtins, '_test_failed'):
+                        builtins._test_failed = True
+                        self.fail("bug")
+        """)
+        testname = self.create_test(code=code)
+
+        output = self.run_tests("-w", testname, exitcode=0)
+        self.check_executed_tests(output, [testname],
+                                  rerun=testname)
+
     def test_no_tests_ran(self):
         code = textwrap.dedent("""
             import unittest
