From 7355149b2c27332697e27d39f53917a8689db2d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Vokr=C3=A1=C4=8Dko?= Date: Wed, 29 Apr 2026 14:10:37 +0200 Subject: [PATCH 1/2] build: measure coverage on source only via coverage run The previous setup relied on pytest-cov (``--cov pytest_split --cov tests`` in addopts) which has a long-standing limitation when measuring a pytest plugin: the plugin imports happen during pytest's plugin discovery, *before* pytest-cov starts tracing. Anything that runs at import time (class bodies, decorators, type aliases, enum definitions) is reported as uncovered even though it obviously executed - the plugin wouldn't load otherwise. The ``CoverageWarning: Module pytest_split was previously imported, but not measured`` made that explicit. The cookiecutter template the project was generated from worked around this by adding ``--cov tests`` to the addopts, inflating the denominator with 100%-covered test files so the average reached 90%. That pulled the number up but stopped reporting the actual source coverage. Switch to ``coverage run -m pytest`` followed by ``coverage report``. Running ``coverage`` from the outside means tracing starts before Python imports anything at all, so plugin import-time code is counted. Real source coverage jumps from ~74% to 99%; threshold goes from 90 to 95 to reflect that. --- .github/workflows/test.yml | 3 ++- pyproject.toml | 12 ++++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f2dbbdd..505b3ad 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,7 +44,8 @@ jobs: - uses: ./.github/actions/python-poetry-env with: python-version: ${{ matrix.python-version }} - - run: poetry run pytest + - run: poetry run coverage run -m pytest + - run: poetry run coverage report docs: runs-on: ubuntu-latest diff --git a/pyproject.toml b/pyproject.toml index be14d7d..164e854 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,16 +63,12 @@ pytest-split = "pytest_split.plugin" target-version = ["py310", "py311", "py312", "py313", "py314"] include = '\.pyi?$' -[tool.pytest.ini_options] -addopts = """\ - --cov pytest_split \ - --cov tests \ - --cov-report term-missing \ - --no-cov-on-fail \ -""" +[tool.coverage.run] +source = ["pytest_split"] [tool.coverage.report] -fail_under = 90 +fail_under = 95 +show_missing = true exclude_lines = [ 'if TYPE_CHECKING:', 'pragma: no cover' From 93b830b152d467b11fe00905017d95fa8a0b4974 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Vokr=C3=A1=C4=8Dko?= Date: Wed, 29 Apr 2026 10:01:47 +0200 Subject: [PATCH 2/2] refactor: assert TestGroup directly in algorithm tests Tests previously asserted each TestGroup field separately (`first.selected`, `first.deselected`, `first.duration`). Consolidate to a single equality check against an expected TestGroup, with concrete values for every field rather than only ``selected``. --- tests/test_algorithms.py | 142 +++++++++++++++++++++++++++++---------- 1 file changed, 105 insertions(+), 37 deletions(-) diff --git a/tests/test_algorithms.py b/tests/test_algorithms.py index a02b6ed..5b388fd 100644 --- a/tests/test_algorithms.py +++ b/tests/test_algorithms.py @@ -10,6 +10,7 @@ from pytest_split.algorithms import ( AlgorithmBase, Algorithms, + TestGroup, ) item = namedtuple("item", "nodeid") # noqa: PYI024 @@ -21,20 +22,25 @@ def test__split_test(self, algo_name): durations = {"a": 1, "b": 1, "c": 1} items = [item(x) for x in durations] algo = Algorithms[algo_name].value + first, second, third = algo(splits=3, items=items, durations=durations) # each split should have one test - assert first.selected == [item("a")] - assert first.deselected == [item("b"), item("c")] - assert first.duration == 1 - - assert second.selected == [item("b")] - assert second.deselected == [item("a"), item("c")] - assert second.duration == 1 - - assert third.selected == [item("c")] - assert third.deselected == [item("a"), item("b")] - assert third.duration == 1 + assert first == TestGroup( + selected=[item("a")], + deselected=[item("b"), item("c")], + duration=1, + ) + assert second == TestGroup( + selected=[item("b")], + deselected=[item("a"), item("c")], + duration=1, + ) + assert third == TestGroup( + selected=[item("c")], + deselected=[item("a"), item("b")], + duration=1, + ) @pytest.mark.parametrize("algo_name", Algorithms.names()) def test__split_tests_handles_tests_in_durations_but_missing_from_items( @@ -43,39 +49,83 @@ def test__split_tests_handles_tests_in_durations_but_missing_from_items( durations = {"a": 1, "b": 1} items = [item(x) for x in ["a"]] algo = Algorithms[algo_name].value - splits = algo(splits=2, items=items, durations=durations) - first, second = splits - assert first.selected == [item("a")] - assert second.selected == [] + first, second = algo(splits=2, items=items, durations=durations) + + assert first == TestGroup( + selected=[item("a")], deselected=[], duration=1 + ) + assert second == TestGroup( + selected=[], deselected=[item("a")], duration=0 + ) @pytest.mark.parametrize("algo_name", Algorithms.names()) def test__split_tests_handles_tests_with_missing_durations(self, algo_name): durations = {"a": 1} items = [item(x) for x in ["a", "b"]] algo = Algorithms[algo_name].value - splits = algo(splits=2, items=items, durations=durations) - first, second = splits - assert first.selected == [item("a")] - assert second.selected == [item("b")] + first, second = algo(splits=2, items=items, durations=durations) + + assert first == TestGroup( + selected=[item("a")], deselected=[item("b")], duration=1 + ) + assert second == TestGroup( + selected=[item("b")], deselected=[item("a")], duration=1 + ) def test__split_test_handles_large_duration_at_end(self): """NOTE: only least_duration does this correctly""" durations = {"a": 1, "b": 1, "c": 1, "d": 3} items = [item(x) for x in ["a", "b", "c", "d"]] algo = Algorithms["least_duration"].value - splits = algo(splits=2, items=items, durations=durations) - first, second = splits - assert first.selected == [item("d")] - assert second.selected == [item(x) for x in ["a", "b", "c"]] + first, second = algo(splits=2, items=items, durations=durations) + + assert first == TestGroup( + selected=[item("d")], + deselected=[item("a"), item("b"), item("c")], + duration=3, + ) + assert second == TestGroup( + selected=[item("a"), item("b"), item("c")], + deselected=[item("d")], + duration=3, + ) @pytest.mark.parametrize( ("algo_name", "expected"), [ - ("duration_based_chunks", [[item("a"), item("b")], [item("c"), item("d")]]), - ("least_duration", [[item("a"), item("c")], [item("b"), item("d")]]), + ( + "duration_based_chunks", + [ + TestGroup( + selected=[item("a"), item("b")], + deselected=[item("c"), item("d")], + duration=2, + ), + TestGroup( + selected=[item("c"), item("d")], + deselected=[item("a"), item("b")], + duration=2, + ), + ], + ), + ( + "least_duration", + [ + TestGroup( + selected=[item("a"), item("c")], + deselected=[item("b"), item("d")], + duration=2, + ), + TestGroup( + selected=[item("b"), item("d")], + deselected=[item("a"), item("c")], + duration=2, + ), + ], + ), ], ) def test__split_tests_calculates_avg_test_duration_only_on_present_tests( @@ -88,23 +138,43 @@ def test__split_tests_calculates_avg_test_duration_only_on_present_tests( durations = {"b": 1, "c": 1, "d": 1, "e": 10000} items = [item(x) for x in ["a", "b", "c", "d"]] algo = Algorithms[algo_name].value - splits = algo(splits=2, items=items, durations=durations) - first, second = splits - expected_first, expected_second = expected - assert first.selected == expected_first - assert second.selected == expected_second + groups = algo(splits=2, items=items, durations=durations) + + assert groups == expected @pytest.mark.parametrize( ("algo_name", "expected"), [ ( "duration_based_chunks", - [[item("a"), item("b"), item("c"), item("d"), item("e")], []], + [ + TestGroup( + selected=[item(x) for x in "abcde"], + deselected=[], + duration=10014, + ), + TestGroup( + selected=[], + deselected=[item(x) for x in "abcde"], + duration=0, + ), + ], ), ( "least_duration", - [[item("e")], [item("a"), item("b"), item("c"), item("d")]], + [ + TestGroup( + selected=[item("e")], + deselected=[item(x) for x in "dcba"], + duration=10000, + ), + TestGroup( + selected=[item(x) for x in "abcd"], + deselected=[item("e")], + duration=14, + ), + ], ), ], ) @@ -112,12 +182,10 @@ def test__split_tests_maintains_relative_order_of_tests(self, algo_name, expecte durations = {"a": 2, "b": 3, "c": 4, "d": 5, "e": 10000} items = [item(x) for x in ["a", "b", "c", "d", "e"]] algo = Algorithms[algo_name].value - splits = algo(splits=2, items=items, durations=durations) - first, second = splits - expected_first, expected_second = expected - assert first.selected == expected_first - assert second.selected == expected_second + groups = algo(splits=2, items=items, durations=durations) + + assert groups == expected def test__split_tests_same_set_regardless_of_order(self): """NOTE: only least_duration does this correctly"""