From 229b2ac5af775440ec0359be18821721b00397fa Mon Sep 17 00:00:00 2001 From: Andrew Gilbert Date: Tue, 22 Jan 2019 17:56:31 -0800 Subject: [PATCH 1/8] Pass a list to `pytest.main`, as is required --- tests/test_ordering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ordering.py b/tests/test_ordering.py index 12f4689..6c9bf52 100644 --- a/tests/test_ordering.py +++ b/tests/test_ordering.py @@ -269,6 +269,6 @@ def test_5(self): pass def test_run_marker_registered(capsys): - pytest.main('--markers') + pytest.main(['--markers']) out, err = capsys.readouterr() assert '@pytest.mark.run' in out From 5ee36328f57069992acfb407f2b3a2d2225f92df Mon Sep 17 00:00:00 2001 From: Andrew Gilbert Date: Tue, 22 Jan 2019 17:58:35 -0800 Subject: [PATCH 2/8] Adding myself to AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index b914557..fba347e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -2,3 +2,4 @@ Frank Tobia Sergei Chipiga Ben Greene Adam Talsma +Andrew Gilbert From 09bb649143a52523c6565ac49e211df009b881ec Mon Sep 17 00:00:00 2001 From: Andrew Gilbert Date: Sat, 19 Jan 2019 20:20:28 -0800 Subject: [PATCH 3/8] Add support for `--indulgent-ordering` option This requests that the plugin be run first, allowing other plugins to override its ordering. This is useful if the order requirements provided by this plugin are intended to be priorities, rather than absolute requirements. For example, if some test cases run slower than others in your test suite, you can mark them as desirable to run first when running tests in parallel, allowing a more efficient use of a multi-core processor. By adding the `--indulgent-ordering` option, this ordering can still be overridden by, for example, the `--failed-first` option, which prioritizes tests which failed during the last run. Without `--indulgent-ordering`, the `--failed-first` option will be overridden by pytest-ordering, reducing its usefulness by increasing the total time until the results are available. --- CHANGELOG | 6 ++++++ docs/source/index.rst | 15 ++++++++++++++ pytest_ordering/__init__.py | 40 +++++++++++++++++++++++++++++++++---- 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index d833b2a..9ec76e3 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,9 @@ +Unreleased +--- +### Added +- #?? Add ``--indulgent-ordering`` to request that the sort from + pytest-ordering be run before other plugins. This allows the built-in + ``--failed-first`` implementation to override the ordering. 0.6 --- diff --git a/docs/source/index.rst b/docs/source/index.rst index 0f5d3da..696bade 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -114,6 +114,21 @@ You can also use markers such as "first", "second", "last", and "second_to_last" =========================== 4 passed in 0.02 seconds =========================== +``--indulgent-ordering`` and overriding ordering +------------- + +You may sometimes find that you want to suggest an ordering of tests, while +allowing it to be overridden for good reson. For example, if you run your test +suite in parallel and have a number of tests which are particularly slow, it +might be desirable to start those tests running first, in order to optimize +your completion time. You can use the pytest-ordering plugin to inform pytest +of this. Now suppose you also want to prioritize tests which failed during the +previous run, by using the ``--failed-first`` option. By default, +pytest-ordering will override the ``--failed-first`` order, but by adding the +``--indulgent-ordering`` option, you can ask pytest to run the sort from +pytest-ordering *before* the sort from ``--failed-first``, allowing the failed +tests to be sorted to the front. + Aspirational ============ diff --git a/pytest_ordering/__init__.py b/pytest_ordering/__init__.py index 0cca91d..4cb541e 100644 --- a/pytest_ordering/__init__.py +++ b/pytest_ordering/__init__.py @@ -26,7 +26,8 @@ def pytest_configure(config): - """Register the "run" marker.""" + """Register the "run" marker and configur the plugin depending on the CLI + options""" config_line = ( 'run: specify ordering information for when tests should run ' @@ -34,9 +35,40 @@ def pytest_configure(config): 'See also: http://pytest-ordering.readthedocs.org/' ) config.addinivalue_line('markers', config_line) - - -def pytest_collection_modifyitems(session, config, items): + if config.getoption('indulgent-ordering'): + # We need to dynamically add this `tryfirst` decorator to the plugin: + # only when the CLI option is present should the decorator be added. + # Thus, we manually run the decorator on the class function and + # manually replace it. + # Python 2.7 didn't allow arbitrary attributes on methods, so we have + # to keep the function as a function and then add it to the class as a + # pseudomethod. Since the class is purely for structuring and `self` + # is never referenced, this seems reasonable. + OrderingPlugin.pytest_collection_modifyitems = pytest.hookimpl( + function=modify_items, tryfirst=True) + else: + OrderingPlugin.pytest_collection_modifyitems = pytest.hookimpl( + function=modify_items, trylast=True) + config.pluginmanager.register(OrderingPlugin(), 'orderingplugin') + + +def pytest_addoption(parser): + """Set up CLI option for pytest""" + group = parser.getgroup('ordering') + group.addoption('--indulgent-ordering', action='store_true', + dest='indulgent-ordering', help='''Request that the sort \ +order provided by pytest-ordering be applied before other sorting, allowing the \ +other sorting to have priority''') + +class OrderingPlugin: + """ + Plugin implementation + + By putting this in a class, we are able to dynamically register it after + the CLI is parsed. + """ + +def modify_items(session, config, items): grouped_items = {} for item in items: From b8102f8b368e31aa6a03c11d469aecfdf2f87411 Mon Sep 17 00:00:00 2001 From: Andrew Gilbert Date: Tue, 22 Jan 2019 21:52:49 -0800 Subject: [PATCH 4/8] Add test for new option --- tests/test_indulgent_ordering.py | 40 ++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 tests/test_indulgent_ordering.py diff --git a/tests/test_indulgent_ordering.py b/tests/test_indulgent_ordering.py new file mode 100644 index 0000000..c58197b --- /dev/null +++ b/tests/test_indulgent_ordering.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +import re + +import pytest +import pytest_ordering + +pytest_plugins = ["pytester"] + + +def test_run_marker_registered(capsys, tmpdir): + testname = str(tmpdir.join("failing.py")) + with open(testname, "w") as fi: + fi.write( + """ +import pytest + +@pytest.mark.first +def test_me_first(): + assert True + +@pytest.mark.second +def test_me_second(): + assert True + +def test_that_fails(): + assert False +""" + ) + args = ["--quiet", "--color=no", testname] + pytest.main(args, [pytest_ordering]) + out, err = capsys.readouterr() + assert "..F" in out + args.insert(0, "--ff") + pytest.main(args, [pytest_ordering]) + out, err = capsys.readouterr() + assert "..F" in out + args.insert(0, "--indulgent-ordering") + pytest.main(args, [pytest_ordering]) + out, err = capsys.readouterr() + assert "F.." in out From 04d8b2e4457ff31938b1ed21385dd15e5867159c Mon Sep 17 00:00:00 2001 From: Andrew Gilbert Date: Tue, 22 Jan 2019 19:46:43 -0800 Subject: [PATCH 5/8] Use correct pull request number in changelog --- CHANGELOG | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 9ec76e3..c1748e8 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,7 +1,7 @@ Unreleased --- ### Added -- #?? Add ``--indulgent-ordering`` to request that the sort from +- #50 Add ``--indulgent-ordering`` to request that the sort from pytest-ordering be run before other plugins. This allows the built-in ``--failed-first`` implementation to override the ordering. From 51ba10b4731df160c01fc10627830ce5168002a2 Mon Sep 17 00:00:00 2001 From: Andrew Gilbert Date: Thu, 24 Jan 2019 19:55:29 -0800 Subject: [PATCH 6/8] Fix spelling --- pytest_ordering/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest_ordering/__init__.py b/pytest_ordering/__init__.py index 4cb541e..bcace9c 100644 --- a/pytest_ordering/__init__.py +++ b/pytest_ordering/__init__.py @@ -26,7 +26,7 @@ def pytest_configure(config): - """Register the "run" marker and configur the plugin depending on the CLI + """Register the "run" marker and configure the plugin depending on the CLI options""" config_line = ( From 9571d4204b50c2bbeab602fb880f00aa5398b574 Mon Sep 17 00:00:00 2001 From: Andrew Gilbert Date: Thu, 24 Jan 2019 19:55:36 -0800 Subject: [PATCH 7/8] Make class Python 2 compatible --- pytest_ordering/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest_ordering/__init__.py b/pytest_ordering/__init__.py index bcace9c..d772754 100644 --- a/pytest_ordering/__init__.py +++ b/pytest_ordering/__init__.py @@ -60,7 +60,7 @@ def pytest_addoption(parser): order provided by pytest-ordering be applied before other sorting, allowing the \ other sorting to have priority''') -class OrderingPlugin: +class OrderingPlugin(object): """ Plugin implementation From 8db0ebe65c6756f8f24a90cd0c90ed762a1163cc Mon Sep 17 00:00:00 2001 From: Andrew Gilbert Date: Thu, 24 Jan 2019 21:26:38 -0800 Subject: [PATCH 8/8] Put tests out of order to check sorting --- tests/test_indulgent_ordering.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_indulgent_ordering.py b/tests/test_indulgent_ordering.py index c58197b..4cc9941 100644 --- a/tests/test_indulgent_ordering.py +++ b/tests/test_indulgent_ordering.py @@ -14,16 +14,16 @@ def test_run_marker_registered(capsys, tmpdir): """ import pytest -@pytest.mark.first -def test_me_first(): - assert True - @pytest.mark.second def test_me_second(): assert True def test_that_fails(): assert False + +@pytest.mark.first +def test_me_first(): + assert True """ ) args = ["--quiet", "--color=no", testname]