- fixed interaction with --continue-on-collection-errors;

- removed sys.exit() based tests, as latest versions of pytest do not
  tolerate that well;

- adjusted test expectations given that different pytest versions
  disagree on the exit code in certain circumstances;

src/pytest_cleanslate/plugin.py

@@ -153,6 +153,10 @@ def pytest_runtest_protocol(self, item: pytest.Item, nextitem: pytest.Item):
             # note any side effects, such as setting session.shouldstop, are lost...
             reports = run_item_forked(item)
 
+        if (reports and isinstance(reports[0], pytest.CollectReport) and reports[0].outcome == 'failed'
+            and not item.config.option.continue_on_collection_errors):
+            item.session.shouldstop = 'collection error'
+
         for rep in reports:
             ihook.pytest_runtest_logreport(report=rep)
 

tests/test_cleanslate.py

@@ -17,8 +17,7 @@ def seq2p(tests_dir, seq):
     'assert': 'assert False',
     'exception': 'raise RuntimeError("test")',
     'kill': 'os.kill(os.getpid(), 9)',
-#    'exit': 'pytest.exit("goodbye")', # FIXME add support
-    'exit': 'sys.exit(0)',
+    'exit': 'pytest.exit("goodbye")',
     'interrupt': 'raise KeyboardInterrupt()'
 }
 
@@ -108,11 +107,16 @@ def test_check_suite_fails(tests_dir, pollute_in_collect, fail_collect, fail_kin
                         fail_collect=fail_collect, fail_kind=fail_kind)
 
     p = subprocess.run([sys.executable, '-m', 'pytest', tests_dir], check=False)
-    if fail_collect or fail_kind == 'interrupt':
+    if fail_collect or fail_kind in ('interrupt', 'exit'):
         assert p.returncode == pytest.ExitCode.INTERRUPTED
     else:
         assert p.returncode == pytest.ExitCode.TESTS_FAILED
 
+    p = subprocess.run([sys.executable, '-m', 'pytest',
+                        '--continue-on-collection-errors', tests_dir], check=False)
+    # pytest versions disagree on what the exit code should be here; more current
+    # ones seem to side with INTERRUPTED for 'interrupt' and 'exit'
+    assert p.returncode in (pytest.ExitCode.INTERRUPTED, pytest.ExitCode.TESTS_FAILED)
 
 @pytest.mark.parametrize("plugin", ['asyncio', 'no:asyncio'])
 @pytest.mark.parametrize("pollute_in_collect, fail_collect", [[False, False], [True, False], [True, True]])
@@ -134,7 +138,7 @@ def test_pytest_discover_tests(tests_dir, fail_kind):
 
 
 @pytest.mark.parametrize("pollute_in_collect, fail_collect", [[False, False], [True, False], [True, True]])
-@pytest.mark.parametrize("fail_kind", list(FAILURES.keys()))
+@pytest.mark.parametrize("fail_kind", list(FAILURES.keys() - {'kill'}))
 def test_unconditionally_failing_test(tests_dir, pollute_in_collect, fail_collect, fail_kind):
     _, _, tests = make_polluted_suite(tests_dir, pollute_in_collect=pollute_in_collect,
                                       fail_collect=fail_collect, fail_kind=fail_kind)
@@ -158,13 +162,24 @@ def test_foo():
     p = subprocess.run([sys.executable, '-m', 'pytest', '--cleanslate', tests_dir], check=False,
                        capture_output=True)
     print(failing.read_text())
-    assert p.returncode == (pytest.ExitCode.INTERRUPTED if (fail_kind == 'interrupt' and not fail_collect)
-                            else pytest.ExitCode.TESTS_FAILED)
-
+    if fail_collect or fail_kind in ('interrupt', 'exit'):
+        assert p.returncode == pytest.ExitCode.INTERRUPTED
+    else:
+        assert p.returncode == pytest.ExitCode.TESTS_FAILED
     # pytest-forked error message that shouldn't be used unless the process was truly killed
     assert 'CRASHED with signal 0' not in str(p.stdout, 'utf-8')
 
 
+    p = subprocess.run([sys.executable, '-m', 'pytest',
+                        '--continue-on-collection-errors', tests_dir],
+                       check=False, capture_output=True)
+    if fail_kind == 'interrupt' or (fail_kind == 'exit' and not fail_collect):
+        assert p.returncode == pytest.ExitCode.INTERRUPTED
+    else:
+        assert p.returncode == pytest.ExitCode.TESTS_FAILED
+    assert 'CRASHED with signal 0' not in str(p.stdout, 'utf-8')
+
+
 def test_collect_failure(tests_dir):
     # _unconditionally_ failing test
     failing = seq2p(tests_dir, 1)
@@ -181,8 +196,8 @@ def test_foo():
 
     p = subprocess.run([sys.executable, '-m', 'pytest', '--cleanslate', tests_dir], check=False,
                        capture_output=True)
-    # FIXME collection errors should show as interrupted
-    assert p.returncode == pytest.ExitCode.TESTS_FAILED
+    print(str(p.stdout, 'utf-8'))
+    assert p.returncode == pytest.ExitCode.INTERRUPTED
 
     # pytest-forked error message that shouldn't be used unless the process was truly killed
     assert 'CRASHED with signal 0' not in str(p.stdout, 'utf-8')