diff --git a/src/coverup/coverup.py b/src/coverup/coverup.py index ef1ea1b..f631d74 100644 --- a/src/coverup/coverup.py +++ b/src/coverup/coverup.py @@ -514,6 +514,7 @@ async def improve_coverage(chatter: llm.Chatter, prompter: prompt.Prompter, seg: pytest_args = (f"--count={args.repeat_tests} " if args.repeat_tests else "") + args.pytest_args coverage = await measure_test_coverage(test=last_test, tests_dir=args.tests_dir, pytest_args=pytest_args, + isolate_tests=args.isolate_tests, branch_coverage=args.branch_coverage, log_write=lambda msg: log_write(seg, msg)) diff --git a/src/coverup/testrunner.py b/src/coverup/testrunner.py index 4f09805..11ae351 100644 --- a/src/coverup/testrunner.py +++ b/src/coverup/testrunner.py @@ -9,7 +9,8 @@ from .utils import subprocess_run -async def measure_test_coverage(*, test: str, tests_dir: Path, pytest_args='', log_write=None, branch_coverage=True): +async def measure_test_coverage(*, test: str, tests_dir: Path, pytest_args='', + log_write=None, isolate_tests=False, branch_coverage=True): """Runs a given test and returns the coverage obtained.""" with tempfile.NamedTemporaryFile(prefix="tmp_test_", suffix='.py', dir=str(tests_dir), mode="w") as t: t.write(test) @@ -20,7 +21,9 @@ async def measure_test_coverage(*, test: str, tests_dir: Path, pytest_args='', l # -qq to cut down on tokens p = await subprocess_run([sys.executable, '-m', 'slipcover', *(('--branch',) if branch_coverage else ()), '--json', '--out', j.name, - '-m', 'pytest', *pytest_args.split(), '-qq', '-x', '--disable-warnings', t.name], + '-m', 'pytest', *pytest_args.split(), + *(('--cleanslate',) if isolate_tests else ()), + '-qq', '-x', '--disable-warnings', t.name], check=True, timeout=60) if log_write: log_write(str(p.stdout, 'UTF-8', errors='ignore')) @@ -37,7 +40,8 @@ async def measure_test_coverage(*, test: str, tests_dir: Path, pytest_args='', l return cov -def measure_suite_coverage(*, tests_dir: Path, source_dir: Path, pytest_args='', trace=None, isolate_tests=False, branch_coverage=True): +def measure_suite_coverage(*, tests_dir: Path, source_dir: Path, pytest_args='', + trace=None, isolate_tests=False, branch_coverage=True): """Runs an entire test suite and returns the coverage obtained.""" with tempfile.NamedTemporaryFile(suffix='.json', delete=False) as j: diff --git a/tests/test_testrunner.py b/tests/test_testrunner.py index b51fe51..47e314f 100644 --- a/tests/test_testrunner.py +++ b/tests/test_testrunner.py @@ -3,6 +3,7 @@ import coverup.testrunner as tr from pathlib import Path import tempfile +import textwrap @pytest.mark.asyncio @@ -11,6 +12,36 @@ async def test_measure_test_coverage_exit_1(tmpdir): await tr.measure_test_coverage(test="import os;\ndef test_foo(): os.exit(1)\n", tests_dir=Path(tmpdir)) +@pytest.mark.asyncio +async def test_measure_test_coverage_uses_cleanslate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + + (tmp_path / "foo.py").write_text(textwrap.dedent("""\ + answer = 42 + """ + )) + test = textwrap.dedent("""\ + import os + + def test_one(): + from foo import answer + assert answer == 42 + + def test_two(monkeypatch): + def mock_stat(path): + if path == '/': + return os.stat_result((0, 0, 0, 0, 0, 0, 0, 0, 0, 0)) + raise Exception('mocked stat failure') + + monkeypatch.setattr(os, 'stat', mock_stat) + from foo import answer # should fail because of 'stat' if run in isolation + assert answer == 42 + """) + + with pytest.raises(subprocess.CalledProcessError) as einfo: + await tr.measure_test_coverage(test=test, tests_dir=tmp_path, isolate_tests=True) + + @pytest.mark.parametrize("absolute", [True, False]) def test_measure_suite_coverage_empty_dir(absolute): with tempfile.TemporaryDirectory(dir=Path('.')) as tests_dir: