- fixed reduce's --trace not being propagated correctly;

- fixed reduce failing if a test module fails collection before the
  test suffering from pollution can run;

src/pytest_cleanslate/reduce.py

@@ -140,6 +140,7 @@ def _run_pytest(tests_path: Path, extra_args=(), *,
     import tempfile
     import subprocess
 
+    # Specifying dir='.' here works around weird failures running tests on MacOS
     with tempfile.TemporaryDirectory(dir='.') as tmpdir:
         tmpdir = Path(tmpdir)
 
@@ -177,7 +178,7 @@ def _bisect_items(items: T.List[str], failing: str, fails: T.Callable[[T.List[st
     assert failing not in items
 
     while len(items) > 1:
-        print(f"    {len(items)}")
+        print(f"... {len(items)}")
         middle = len(items) // 2
 
         if fails(items[:middle]+[failing]):
@@ -194,13 +195,13 @@ def _bisect_items(items: T.List[str], failing: str, fails: T.Callable[[T.List[st
     if len(items) == 1 and fails([failing]):
         items = []
 
-    print(f"    {len(items)}")
+    print(f"... {len(items)}")
     return items
 
 
 def _reduce_tests(tests_path: Path, tests: T.List[str], failing_test: str, *, trace=None) -> T.List[str]:
     def fails(test_set: T.List[str]):
-        trial = _run_pytest(tests_path, ('-x',), tests=test_set, trace=trace)
+        trial = _run_pytest(tests_path, ('--continue-on-collection-errors',), tests=test_set, trace=trace)
         return trial.get_outcome(failing_test) == 'failed'
 
     return _bisect_items(tests, failing_test, fails)
@@ -209,7 +210,7 @@ def fails(test_set: T.List[str]):
 def _reduce_modules(tests_path: Path, tests: T.List[str], failing_test: str,
                     modules: T.List[str], failing_module: str, *, trace=None) -> T.List[str]:
     def fails(module_set: T.List[str]):
-        trial = _run_pytest(tests_path, ('-x',), tests=tests, modules=module_set, trace=trace)
+        trial = _run_pytest(tests_path, ('--continue-on-collection-errors',), tests=tests, modules=module_set, trace=trace)
         return trial.get_outcome(failing_test) == 'failed'
 
     return _bisect_items(modules, failing_module, fails)
@@ -253,13 +254,13 @@ def main():
         if args.trace: print()
         print(f"Module \"{failed}\"'s collection failed; trying it by itself...", flush=True)
         failed_module = failed
-        solo = _run_pytest(args.tests_path, ('-x',), modules=[failed_module], trace=args.trace)
+        solo = _run_pytest(args.tests_path, modules=[failed_module], trace=args.trace)
     else:
         if args.trace: print()
         print(f"Test \"{failed}\" failed; trying it by itself...", flush=True)
         failed_module = results.get_module(failed)
 
-        solo = _run_pytest(args.tests_path, ('-x',), modules=[failed_module], tests=[failed], trace=args.trace)
+        solo = _run_pytest(args.tests_path, modules=[failed_module], tests=[failed], trace=args.trace)
 
     if solo.get_outcome(failed) != 'passed':
         print("That also fails by itself!", flush=True)
@@ -278,13 +279,13 @@ def main():
 
         if args.trace: print()
         print("Trying to reduce test set...", flush=True)
-        tests = _reduce_tests(args.tests_path, tests, failed)
+        tests = _reduce_tests(args.tests_path, tests, failed, trace=args.trace)
 
     if args.trace: print()
     print("Trying to reduce module set...", flush=True)
 
     modules = [m for m in results.get_modules() if m != failed_module]
-    modules = _reduce_modules(args.tests_path, tests if is_module else tests + [failed], failed, modules, failed_module)
+    modules = _reduce_modules(args.tests_path, tests if is_module else tests + [failed], failed, modules, failed_module, trace=args.trace)
 
     if args.trace: print()
     print("Reduced failure set:")

tests/test_cleanslate.py

@@ -22,13 +22,16 @@ def seq2p(tests_dir, seq):
 }
 
 def make_polluted_suite(tests_dir: Path, *, pollute_in_collect: bool = True, fail_collect: bool = False,
-                        fail_kind: str = 'assert'):
+                        fail_kind: str = 'assert', polluter_seq: int = None, failing_seq: int = None):
     """in a suite with 10 tests, 'polluter' doesn't fail, but causes 'failing' to fail."""
-    tests = list(range(10))
-
     # note the polluter must run before the failing test
+    N_TESTS = 10
+
+    assert (polluter_seq is None and failing_seq is None) or (failing_seq > polluter_seq)
+
+    if polluter_seq is None:
+        polluter_seq = random.choice(range(N_TESTS-1))
 
-    polluter_seq = tests.pop(random.choice(tests[:-1]))
     polluter = seq2p(tests_dir, polluter_seq)
     if pollute_in_collect:
         polluter.write_text(dedent("""\
@@ -51,7 +54,10 @@ def test_bar():
             """))
         polluter = f"{polluter}::test_polluter"
 
-    failing = seq2p(tests_dir, tests.pop(random.choice(range(polluter_seq, len(tests)))))
+    if failing_seq is None:
+        failing_seq = random.choice(range(polluter_seq+1, N_TESTS))
+
+    failing = seq2p(tests_dir, failing_seq)
     if fail_collect:
         failing.write_text(dedent(f"""\
             import sys
@@ -79,6 +85,7 @@ def test_ok():
             """))
         failing = f"{failing}::test_failing"
 
+    tests = [seq for seq in range(N_TESTS) if seq not in (polluter_seq, failing_seq)]
     for seq in tests:
         seq2p(tests_dir, seq).write_text('def test_foo(): pass')
 

tests/test_reduce.py

@@ -180,3 +180,41 @@ def test_reduce(tests_dir, pollute_in_collect, fail_collect):
     assert reduction['failed'] == failing
     assert reduction['modules'] == [get_test_module(polluter)]
     assert reduction['tests'] == [] if pollute_in_collect else [polluter]
+
+
+def test_reduce_other_collection_fails(tests_dir):
+    """Tests that we use --continue-on-collection-errors"""
+    failing, polluter, tests = make_polluted_suite(tests_dir, pollute_in_collect=True, fail_collect=False,
+                                                   polluter_seq = 3, failing_seq = 8)
+
+    seq2p(tests_dir, 0).write_text(dedent("""\
+        import sys
+        sys.needs_this = True
+
+        def test_nothing():
+            assert True
+        """))
+
+    # this one fails collection pytest runs set to ignore the one above
+    seq2p(tests_dir, 2).write_text(dedent("""\
+        import sys
+
+        if not hasattr(sys, 'needs_this'):
+            raise RuntimeError('argh')
+
+        def test_nothing():
+            assert True
+        """))
+
+    reduction_file = tests_dir.parent / "reduction.json"
+
+    p = subprocess.run([sys.executable, '-m', 'pytest_cleanslate.reduce',
+                        '--save-to', reduction_file, '--trace', tests_dir], check=False)
+    assert p.returncode == 0
+
+    with reduction_file.open("r") as f:
+        reduction = json.load(f)
+
+    assert reduction['failed'] == failing
+    assert reduction['modules'] == [get_test_module(polluter)]
+    assert reduction['tests'] == []