Skip to content

Commit

Permalink
Merge pull request #331 from Akrog/preserve-order
Browse files Browse the repository at this point in the history
Fix #330 - Preserve load-list order
  • Loading branch information
mtreinish authored Aug 31, 2022
2 parents 2191ea8 + ff25c99 commit 6edb3ed
Show file tree
Hide file tree
Showing 9 changed files with 266 additions and 53 deletions.
27 changes: 16 additions & 11 deletions stestr/commands/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,21 @@ def gather_errors(test_dict):
return ids


def _load_list_and_filter(list_filename, filter_ids):
"""Load list of IDs and optionally filter them by a list of ids."""
list_ids = []
set_ids = set()
# Should perhaps be text.. currently does its own decode.
with open(list_filename, 'rb') as list_file:
# Remove duplicates and filter out non failing tests when required
for test_id in parse_list(list_file.read()):
if test_id not in set_ids and (filter_ids is None or
test_id in filter_ids):
list_ids.append(test_id)
set_ids.add(test_id)
return list_ids


def run_command(config='.stestr.conf',
repo_url=None, test_path=None, top_dir=None, group_regex=None,
failing=False, serial=False, concurrency=0, load_list=None,
Expand Down Expand Up @@ -492,17 +507,7 @@ def run_tests():
else:
ids = None
if load_list:
list_ids = set()
# Should perhaps be text.. currently does its own decode.
with open(load_list, 'rb') as list_file:
list_ids = set(parse_list(list_file.read()))
if ids is None:
# Use the supplied list verbatim
ids = list_ids
else:
# We have some already limited set of ids, just reduce to ids
# that are both failing and listed.
ids = list_ids.intersection(ids)
ids = _load_list_and_filter(load_list, ids)

if config and os.path.isfile(config):
conf = config_file.TestrConf(config)
Expand Down
36 changes: 17 additions & 19 deletions stestr/selection.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
# under the License.

import contextlib
import random
import re
import sys

Expand Down Expand Up @@ -89,7 +90,7 @@ def _get_regex_from_include_list(file_path):


def construct_list(test_ids, regexes=None, exclude_list=None,
include_list=None, exclude_regex=None):
include_list=None, exclude_regex=None, randomize=False):
"""Filters the discovered test cases
:param list test_ids: The set of test_ids to be filtered
Expand All @@ -100,10 +101,11 @@ def construct_list(test_ids, regexes=None, exclude_list=None,
:param str exclude_list: The path to an exclusion_list file
:param str include_list: The path to an inclusion_list file
:param str exclude_regex: regex pattern to exclude tests
:param str randomize: Randomize the result
:return: iterable of strings. The strings are full
test_ids
:rtype: set
:rtype: list
"""

if not regexes:
Expand Down Expand Up @@ -137,20 +139,16 @@ def construct_list(test_ids, regexes=None, exclude_list=None,
exclude_data = [record]

list_of_test_cases = filter_tests(regexes, test_ids)
set_of_test_cases = set(list_of_test_cases)

if not exclude_data:
return set_of_test_cases

# NOTE(afazekas): We might use a faster logic when the
# print option is not requested
for (rex, msg, s_list) in exclude_data:
for test_case in list_of_test_cases:
if rex.search(test_case):
# NOTE(mtreinish): In the case of overlapping regex the test
# case might have already been removed from the set of tests
if test_case in set_of_test_cases:
set_of_test_cases.remove(test_case)
s_list.append(test_case)

return set_of_test_cases

if exclude_data:
# NOTE(afazekas): We might use a faster logic when the
# print option is not requested
for (rex, msg, s_list) in exclude_data:
# NOTE(mtreinish): In the case of overlapping regex the test case
# might have already been removed from the set of tests
list_of_test_cases = [tc for tc in list_of_test_cases
if not rex.search(tc)]
if randomize:
random.shuffle(list_of_test_cases)

return list_of_test_cases
47 changes: 28 additions & 19 deletions stestr/subunit_runner/program.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@


def filter_by_ids(suite_or_case, test_ids):
"""Remove tests from suite_or_case where their id is not in test_ids.
"""Return a test suite with the intersection of suite_or_case and test_ids.
:param suite_or_case: A test suite or test case.
:param test_ids: Something that supports the __contains__ protocol.
Expand Down Expand Up @@ -58,23 +58,31 @@ def filter_by_ids(suite_or_case, test_ids):
thus the backwards-compatible code paths attempt to mutate in place rather
than guessing how to reconstruct a new suite.
"""
# Compatible objects
if extras.safe_hasattr(suite_or_case, 'filter_by_ids'):
return suite_or_case.filter_by_ids(test_ids)
# TestCase objects.
if extras.safe_hasattr(suite_or_case, 'id'):
if suite_or_case.id() in test_ids:
return suite_or_case
else:
return unittest.TestSuite()
# Standard TestSuites or derived classes [assumed to be mutable].
if isinstance(suite_or_case, unittest.TestSuite):
filtered = []
for item in suite_or_case:
filtered.append(filter_by_ids(item, test_ids))
suite_or_case._tests[:] = filtered
# Everything else:
return suite_or_case
def _filter_tests(to_filter, id_map):
# Compatible objects
if extras.safe_hasattr(to_filter, 'filter_by_ids'):
res = to_filter.filter_by_ids(test_ids)
_filter_tests(res, id_map)
# TestCase objects.
elif extras.safe_hasattr(to_filter, 'id'):
test_id = to_filter.id()
if test_id in test_ids:
id_map[test_id] = to_filter
# Standard TestSuites or derived classes.
elif isinstance(to_filter, unittest.TestSuite):
for item in to_filter:
_filter_tests(item, id_map)

id_map = {}
_filter_tests(suite_or_case, id_map)

result = unittest.TestSuite()
for test_id in test_ids:
if test_id in id_map:
# Use pop to avoid duplicates
result.addTest(id_map.pop(test_id))

return result


def iterate_tests(test_suite_or_case):
Expand Down Expand Up @@ -178,8 +186,9 @@ def __init__(self, module='__main__', defaultTest=None, argv=None,
lines = source.readlines()
finally:
source.close()
test_ids = {line.strip().decode('utf-8') for line in lines}
test_ids = [line.strip().decode('utf-8') for line in lines]
self.test = filter_by_ids(self.test, test_ids)

# XXX: Local edit (see http://bugs.python.org/issue22860)
if not self.listtests:
self.runTests()
Expand Down
6 changes: 4 additions & 2 deletions stestr/test_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,6 @@ def __init__(self, test_ids, cmd_template, listopt, idoption,
self._listpath = listpath
self.test_filters = test_filters
self._group_callback = group_callback
self.worker_path = None
self.worker_path = worker_path
self.concurrency_value = concurrency
self.exclude_list = exclude_list
Expand Down Expand Up @@ -145,11 +144,14 @@ def list_subst(match):
name = ''
idlist = ''
else:
# Randomize now if it's not going to be partitioned
randomize = self.randomize and self.concurrency == 1
self.test_ids = selection.construct_list(
self.test_ids, exclude_list=self.exclude_list,
include_list=self.include_list,
regexes=self.test_filters,
exclude_regex=self.exclude_regex)
exclude_regex=self.exclude_regex,
randomize=randomize)
name = self.make_listfile()
variables['IDFILE'] = name
idlist = ' '.join(self.test_ids)
Expand Down
Empty file.
124 changes: 124 additions & 0 deletions stestr/tests/subunit_runner/test_program.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import unittest

from stestr.subunit_runner import program
from stestr.tests import base


class TestGetById(base.TestCase):
class CompatibleSuite(unittest.TestSuite):
def filter_by_ids(self, test_ids):
return unittest.TestSuite([item for item in self
if item.id() in test_ids])

@classmethod
def setUpClass(cls):
cls.test_suite = unittest.TestSuite([cls(name) for name in dir(cls)
if name.startswith('test')])

@classmethod
def _test_name(cls, name):
return f'{__name__}.{cls.__name__}.{name}'

@staticmethod
def _test_ids(suite):
return [item.id() for item in suite]

def test_filter_by_ids_case_no_tests(self):
test_case = type(self)('test_filter_by_ids_case_no_tests')
result = program.filter_by_ids(test_case, [
'stestr.tests.test_load.TestLoadCommand.test_empty_with_pretty_out'
])
self.assertEqual([], self._test_ids(result))

def test_filter_by_ids_case_simple(self):
name = 'test_filter_by_ids_case_simple'
test_name = self._test_name(name)
test_case = type(self)(name)
result = program.filter_by_ids(test_case, [test_name])
self.assertEqual([test_name], self._test_ids(result))

def test_filter_by_ids_suite_no_tests(self):
result = program.filter_by_ids(self.test_suite, [
'stestr.tests.test_load.TestLoadCommand.test_empty_with_pretty_out'
])
self.assertEqual([], self._test_ids(result))

def test_filter_by_ids_suite_simple(self):
test_name = self._test_name('test_filter_by_ids_suite_simple')
result = program.filter_by_ids(self.test_suite, [test_name])
self.assertEqual([test_name], self._test_ids(result))

def test_filter_by_ids_suite_preserves_order(self):
names = ('test_filter_by_ids_suite_simple',
'test_filter_by_ids_suite_preserves_order',
'test_filter_by_ids_suite_no_tests')
test_names = [self._test_name(name) for name in names]
result = program.filter_by_ids(self.test_suite, test_names)
self.assertEqual(test_names, self._test_ids(result))

def test_filter_by_ids_suite_no_duplicates(self):
names = ('test_filter_by_ids_suite_simple',
'test_filter_by_ids_suite_preserves_order')
expected = [self._test_name(name) for name in names]
# Create duplicates in reversed order
test_names = expected + expected[::-1]
result = program.filter_by_ids(self.test_suite, test_names)
self.assertEqual(expected, self._test_ids(result))

def test_filter_by_ids_compatible_no_tests(self):
test_suite = self.CompatibleSuite(self.test_suite)
result = program.filter_by_ids(test_suite, [
'stestr.tests.test_load.TestLoadCommand.test_empty_with_pretty_out'
])
self.assertEqual([], self._test_ids(result))

def test_filter_by_ids_compatible_simple(self):
test_suite = self.CompatibleSuite(self.test_suite)
test_name = self._test_name('test_filter_by_ids_compatible_simple')
result = program.filter_by_ids(test_suite, [test_name])
self.assertEqual([test_name], self._test_ids(result))

def test_filter_by_ids_compatible_preserves_order(self):
test_suite = self.CompatibleSuite(self.test_suite)
names = ('test_filter_by_ids_compatible_simple',
'test_filter_by_ids_compatible_preserves_order',
'test_filter_by_ids_compatible_no_tests')
test_names = [self._test_name(name) for name in names]
result = program.filter_by_ids(test_suite, test_names)
self.assertEqual(test_names, self._test_ids(result))

def test_filter_by_ids_compatible_no_duplicates(self):
test_suite = self.CompatibleSuite(self.test_suite)
names = ('test_filter_by_ids_compatible_simple',
'test_filter_by_ids_compatible_preserves_order')
expected = [self._test_name(name) for name in names]
# Create duplicates in reversed order
test_names = expected + expected[::-1]
result = program.filter_by_ids(test_suite, test_names)
self.assertEqual(expected, self._test_ids(result))

def test_filter_by_ids_no_duplicates_preserve_order(self):
test_suite = unittest.TestSuite(
[self.CompatibleSuite(self.test_suite),
self.test_suite,
type(self)('test_filter_by_ids_no_duplicates_preserve_order')])
names = ('test_filter_by_ids_compatible_simple',
'test_filter_by_ids_suite_simple',
'test_filter_by_ids_compatible_preserves_order')
expected = [self._test_name(name) for name in names]
# Create duplicates in reversed order
test_names = expected + expected[::-1]
result = program.filter_by_ids(test_suite, test_names)
self.assertEqual(expected, self._test_ids(result))
3 changes: 3 additions & 0 deletions stestr/tests/test_load.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,14 @@
import io

from stestr.commands import load
from stestr import subunit_trace
from stestr.tests import base


class TestLoadCommand(base.TestCase):
def test_empty_with_pretty_out(self):
# Clear results that may be there
subunit_trace.RESULTS.clear()
stream = io.BytesIO()
output = io.BytesIO()
res = load.load(in_streams=[('subunit', stream)], pretty_out=True,
Expand Down
Loading

0 comments on commit 6edb3ed

Please sign in to comment.