Skip to content

Commit

Permalink
- added flag to disable using branch coverage, to work around
Browse files Browse the repository at this point in the history
  issues seen with SlipCover's branch coverage implementation
  on Python 3.12;
  • Loading branch information
jaltmayerpizzorno committed Apr 15, 2024
1 parent 103cf1b commit 41614e6
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 8 deletions.
11 changes: 10 additions & 1 deletion src/coverup/coverup.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,10 @@ def default_model():
action=argparse.BooleanOptionalAction,
help='add (parent of) source directory to PYTHONPATH')

ap.add_argument('--branch-coverage', default=True,
action=argparse.BooleanOptionalAction,
help=argparse.SUPPRESS)

def positive_int(value):
ivalue = int(value)
if ivalue < 0: raise argparse.ArgumentTypeError("must be a number >= 0")
Expand Down Expand Up @@ -216,6 +220,7 @@ def check_whole_suite() -> None:
print("Checking test suite... ", end='', flush=True)
try:
btf = BadTestsFinder(tests_dir=args.tests_dir, pytest_args=pytest_args,
branch_coverage=args.branch_coverage,
trace=(print if args.debug else None))
outcomes = btf.run_tests()
failing_tests = list(p for p, o in outcomes.items() if o == 'failed')
Expand All @@ -240,6 +245,7 @@ def print_noeol(message):

try:
btf = BadTestsFinder(tests_dir=args.tests_dir, pytest_args=args.pytest_args,
branch_coverage=args.branch_coverage,
trace=(print if args.debug else None),
progress=(print if args.debug else print_noeol))

Expand Down Expand Up @@ -561,6 +567,7 @@ def log_prompts(prompts: T.List[dict]):

try:
result = await measure_test_coverage(test=last_test, tests_dir=args.tests_dir, pytest_args=args.pytest_args,
branch_coverage=args.branch_coverage,
log_write=lambda msg: log_write(seg, msg))

except subprocess.TimeoutExpired:
Expand All @@ -578,7 +585,7 @@ def log_prompts(prompts: T.List[dict]):

new_lines = set(result[seg.filename]['executed_lines']) if seg.filename in result else set()
new_branches = set(tuple(b) for b in result[seg.filename]['executed_branches']) \
if seg.filename in result else set()
if (seg.filename in result and 'executed_branches' in result[seg.filename]) else set()
now_missing_lines = seg.missing_lines - new_lines
now_missing_branches = seg.missing_branches - new_branches

Expand Down Expand Up @@ -691,6 +698,7 @@ def main():
coverage = measure_suite_coverage(tests_dir=args.tests_dir, source_dir=args.source_dir,
pytest_args=args.pytest_args,
isolate_tests=args.isolate_tests,
branch_coverage=args.branch_coverage,
trace=(print if args.debug else None))
state = State(coverage)

Expand Down Expand Up @@ -777,6 +785,7 @@ async def sem_coro(coro):
coverage = measure_suite_coverage(tests_dir=args.tests_dir, source_dir=args.source_dir,
pytest_args=args.pytest_args,
isolate_tests=args.isolate_tests,
branch_coverage=args.branch_coverage,
trace=(print if args.debug else None))

except subprocess.CalledProcessError as e:
Expand Down
18 changes: 11 additions & 7 deletions src/coverup/testrunner.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from .utils import subprocess_run


async def measure_test_coverage(*, test: str, tests_dir: Path, pytest_args='', log_write=None):
async def measure_test_coverage(*, test: str, tests_dir: Path, pytest_args='', log_write=None, branch_coverage=True):
"""Runs a given test and returns the coverage obtained."""
with tempfile.NamedTemporaryFile(prefix="tmp_test_", suffix='.py', dir=str(tests_dir), mode="w") as t:
t.write(test)
Expand All @@ -20,7 +20,8 @@ async def measure_test_coverage(*, test: str, tests_dir: Path, pytest_args='', l
with tempfile.NamedTemporaryFile(suffix='.json', delete=False) as j:
try:
# -qq to cut down on tokens
p = await subprocess_run([sys.executable, '-m', 'slipcover', '--branch', '--json', '--out', j.name,
p = await subprocess_run([sys.executable, '-m', 'slipcover'] + (['--branch'] if branch_coverage else []) + \
['--json', '--out', j.name,
'-m', 'pytest'] + pytest_args.split() + ['-qq', '-x', '--disable-warnings', t.name],
check=True, timeout=60)
if log_write:
Expand All @@ -38,14 +39,15 @@ async def measure_test_coverage(*, test: str, tests_dir: Path, pytest_args='', l
return cov["files"]


def measure_suite_coverage(*, tests_dir: Path, source_dir: Path, pytest_args='', trace=None, isolate_tests=False):
def measure_suite_coverage(*, tests_dir: Path, source_dir: Path, pytest_args='', trace=None, isolate_tests=False, branch_coverage=True):
"""Runs an entire test suite and returns the coverage obtained."""

with tempfile.NamedTemporaryFile(suffix='.json', delete=False) as j:
try:
command = [sys.executable,
'-m', 'slipcover', '--source', source_dir, '--branch', '--json', '--out', j.name] + \
(['--isolate-tests'] if isolate_tests else []) +\
'-m', 'slipcover', '--source', source_dir] + (['--branch'] if branch_coverage else []) + \
['--json', '--out', j.name] + \
(['--isolate-tests'] if isolate_tests else []) + \
['-m', 'pytest'] + pytest_args.split() + ['-qq', '--disable-warnings', '-x', tests_dir]

if trace: trace(command)
Expand Down Expand Up @@ -75,7 +77,7 @@ class BadTestFinderError(Exception):

class BadTestsFinder(DeltaDebugger):
"""Finds tests that cause other tests to fail."""
def __init__(self, *, tests_dir: Path, pytest_args: str = '', trace = None, progress = None):
def __init__(self, *, tests_dir: Path, pytest_args: str = '', trace = None, progress = None, branch_coverage=True):
super(BadTestsFinder, self).__init__(trace=trace)
self.tests_dir = tests_dir

Expand All @@ -90,6 +92,7 @@ def find_tests(p):

self.all_tests = set(find_tests(self.tests_dir))
self.pytest_args = pytest_args.split()
self.branch_coverage = branch_coverage
self.progress = progress


Expand Down Expand Up @@ -127,7 +130,8 @@ def to_tmpdir(p: Path):
command = [sys.executable,
# include SlipCover in case a test is failing because of it
# (in measure_suite_coverage)
'-m', 'slipcover', '--branch', '--out', '/dev/null',
'-m', 'slipcover'] + (['--branch'] if self.branch_coverage else []) + \
['--out', '/dev/null',
"-m", "pytest"] + self.pytest_args + \
['-qq', '--disable-warnings',
'-p', 'coverup.pytest_plugin', '--coverup-outcomes', str(outcomes_f.name)] \
Expand Down

0 comments on commit 41614e6

Please sign in to comment.