From c9f15e48aac0773240030b94237fe2cd2696986e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9amus=20=C3=93=20Ceanainn?= Date: Sun, 15 Nov 2026 22:11:34 +0000 Subject: [PATCH] Add --no-module-splits with pre-collection filtering --- CHANGELOG.md | 3 + README.md | 21 +- src/pytest_split/algorithms.py | 74 ++++- src/pytest_split/plugin.py | 178 +++++++++++- tests/test_module_partition.py | 485 +++++++++++++++++++++++++++++++++ 5 files changed, 756 insertions(+), 5 deletions(-) create mode 100644 tests/test_module_partition.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1391dff..b0b55e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Add `--no-module-splits` for both splitting algorithms to keep modules together and skip importing and collecting modules assigned to other groups. + ### Fixed - Fix malformed bullet points rendering in GitHub Pages documentation diff --git a/README.md b/README.md index 03d30e2..c791580 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,25 @@ However, when there are major changes in the suite compared to what's stored in The splitting algorithm can be controlled with the `--splitting-algorithm` CLI option and defaults to `duration_based_chunks`. For more information about the different algorithms and their tradeoffs, please see the section below. +### Input arguments + +| Argument | Description | +|----------|-------------| +| `--store-durations` | Store test durations for future runs. | +| `--durations-path PATH` | Path to the durations file. Defaults to `.test_durations` in the current working directory. | +| `--splits N` | Number of groups to split the tests into. Use with `--group`. | +| `--group N` | Group to run, numbered from 1 to `--splits`. | +| `--splitting-algorithm ALGORITHM` | Choose `duration_based_chunks` (default) or `least_duration`. | +| `--no-module-splits` | Keep each module's tests in the same group and skip importing and collecting modules assigned to other groups. Works with either splitting algorithm. | +| `--clean-durations` | Remove stored timings for tests absent from the current run when used with `--store-durations`. | + +With `--no-module-splits`, durations are aggregated per module; modules without timings use the average known module duration. +Group assignment happens before collection, using sorted module paths, so test-order randomization does not change which group owns each module. + +```sh +pytest --splits 3 --group 1 --no-module-splits --splitting-algorithm least_duration +``` + ### CLI commands #### slowest-tests Lists the slowest tests based on the information stored in the test durations file. See `slowest-tests --help` for more @@ -60,7 +79,7 @@ Lists the slowest tests based on the information stored in the test durations fi ## Interactions with other pytest plugins * [`pytest-random-order`](https://github.com/jbasko/pytest-random-order) and [`pytest-randomly`](https://github.com/pytest-dev/pytest-randomly): - ⚠️ `pytest-split` running with the `duration_based_chunks` algorithm is **incompatible** with test-order-randomization plugins. + ⚠️ `pytest-split` running with the `duration_based_chunks` algorithm without `--no-module-splits` is **incompatible** with test-order-randomization plugins. Test selection in the groups happens after randomization, potentially causing some tests to be selected in several groups and others not at all. Instead, a global random seed needs to be computed before running the tests (for example using `$RANDOM` from the shell) and that single seed then needs to be used for all groups by setting the `--random-order-seed` option. diff --git a/src/pytest_split/algorithms.py b/src/pytest_split/algorithms.py index 83223b3..f72a0bd 100644 --- a/src/pytest_split/algorithms.py +++ b/src/pytest_split/algorithms.py @@ -2,7 +2,7 @@ import heapq from abc import ABC, abstractmethod from operator import itemgetter -from typing import TYPE_CHECKING, NamedTuple +from typing import TYPE_CHECKING, NamedTuple, cast if TYPE_CHECKING: from _pytest import nodes @@ -148,6 +148,78 @@ def __call__( ] +class _Module(NamedTuple): + nodeid: str + + def __str__(self) -> str: + return self.nodeid + + +class ModuleGrouping: + """Apply a splitting algorithm to whole modules before test collection.""" + + def __init__(self, algorithm: AlgorithmBase) -> None: + self.algorithm = algorithm + + def module_groups( + self, splits: int, modules: "set[str]", durations: "dict[str, float]" + ) -> "dict[str, int]": + module_durations: dict[str, float] = {} + for nodeid, duration in durations.items(): + path = nodeid.split("::", 1)[0] + if path in modules: + module_durations[path] = module_durations.get(path, 0) + duration + + # Both algorithms only read nodeids and string representations. Lightweight + # module representatives let them run without importing test modules. + representatives = cast( + "list[nodes.Item]", [_Module(path) for path in sorted(modules)] + ) + if not any(module_durations.values()): + # Equal positive weights also handle cached zero-duration modules. + module_durations = dict.fromkeys(modules, 1.0) + groups = self.algorithm(splits, representatives, module_durations) + return { + module.nodeid: group_idx + for group_idx, group in enumerate(groups) + for module in group.selected + } + + def __call__( + self, + splits: int, + items: "list[nodes.Item]", + durations: "dict[str, float]", + *, + module_groups: "dict[str, int] | None" = None, + ) -> "list[TestGroup]": + modules = {item.nodeid.split("::", 1)[0] for item in items} + if module_groups is None: + module_groups = self.module_groups(splits, modules, durations) + else: + # Preserve the pre-collection plan. Modules generated by custom + # collectors are collected on every runner and split separately. + unplanned = self.module_groups( + splits, modules - module_groups.keys(), durations + ) + module_groups = {**module_groups, **unplanned} + selected: list[list[nodes.Item]] = [[] for _ in range(splits)] + deselected: list[list[nodes.Item]] = [[] for _ in range(splits)] + duration: list[float] = [0 for _ in range(splits)] + + for item, item_duration in _get_items_with_durations(items, durations): + group_idx = module_groups[item.nodeid.split("::", 1)[0]] + selected[group_idx].append(item) + duration[group_idx] += item_duration + for i in range(splits): + if i != group_idx: + deselected[i].append(item) + + return [ + TestGroup(selected[i], deselected[i], duration[i]) for i in range(splits) + ] + + def _get_items_with_durations( items: "list[nodes.Item]", durations: "dict[str, float]" ) -> "list[tuple[nodes.Item, float]]": diff --git a/src/pytest_split/plugin.py b/src/pytest_split/plugin.py index 9140936..0929426 100644 --- a/src/pytest_split/plugin.py +++ b/src/pytest_split/plugin.py @@ -1,15 +1,20 @@ +import importlib.util import json import os +from pathlib import Path from typing import TYPE_CHECKING import pytest from _pytest.config import create_terminal_writer, hookimpl +from _pytest.pathlib import fnmatch_ex from _pytest.reports import TestReport from pytest_split import algorithms from pytest_split.ipynb_compatibility import ensure_ipynb_compatibility if TYPE_CHECKING: + from collections.abc import Iterator + from _pytest import nodes from _pytest.config import Config from _pytest.config.argparsing import Parser @@ -63,6 +68,14 @@ def pytest_addoption(parser: "Parser") -> None: default="duration_based_chunks", choices=algorithms.Algorithms.names(), ) + group.addoption( + "--no-module-splits", + action="store_true", + help=( + "Keep each module's tests in the same group and skip importing and " + "collecting modules assigned to other groups." + ), + ) group.addoption( "--clean-durations", dest="clean_durations", @@ -113,6 +126,131 @@ def pytest_configure(config: "Config") -> None: ) +def _ignore_module(path: Path, config: "Config") -> "bool | None": + if ( + config.option.no_module_splits + and config.option.splits + and config.option.group + and path.is_file() + ): + plugin = config.pluginmanager.get_plugin("pytestsplitplugin") + if not isinstance(plugin, PytestSplitPlugin): + return None + group_idx = plugin.module_groups.get(_relative_path(path, config)) + if group_idx is not None and group_idx != config.option.group - 1: + return True + return None + + +def _relative_path(path: Path, config: "Config") -> str: + root_path = getattr(config, "rootpath", None) + if root_path is None: # pytest < 6.1 + root_path = Path(str(getattr(config, "rootdir"))) # noqa: B009 + return Path(os.path.relpath(path, root_path)).as_posix() + + +def _collection_path(argument: str, config: "Config") -> Path: + name = argument.split("::", 1)[0] + path = config.invocation_params.dir / name + if path.exists(): + return path + if config.option.pyargs: + try: + spec = importlib.util.find_spec(name) + except (ImportError, ValueError): + spec = None + if spec is not None: + if spec.submodule_search_locations: + return Path(next(iter(spec.submodule_search_locations))) + if spec.origin is not None: + return Path(spec.origin) + return path + + +def _module_paths(config: "Config", durations: "dict[str, float]") -> "set[str]": + """Discover candidate paths without importing test modules.""" + patterns = [*config.getini("python_files"), "*.ipynb"] + patterns.extend(config.getoption("doctestglob", default=[]) or ["test*.txt"]) + if config.getoption("doctestmodules", default=False): + patterns.append("*.py") + cached_modules = {nodeid.split("::", 1)[0] for nodeid in durations} + modules: set[str] = set() + invocation_dir = config.invocation_params.dir + ignored = [ + invocation_dir / path for path in config.getoption("ignore", default=[]) or [] + ] + ignored_globs = [ + str(invocation_dir / pattern) + for pattern in config.getoption("ignore_glob", default=[]) or [] + ] + for arg in config.args: + path = _collection_path(arg, config) + if path.is_file(): + modules.add(_relative_path(path, config)) + elif path.is_dir(): + modules.update( + _directory_modules( + path, config, patterns, cached_modules, ignored, ignored_globs + ) + ) + return modules + + +def _directory_modules( # noqa: PLR0913 + path: Path, + config: "Config", + patterns: "list[str]", + cached_modules: "set[str]", + ignored: "list[Path]", + ignored_globs: "list[str]", +) -> "Iterator[str]": + for directory, subdirs, names in os.walk(path): + current = Path(directory) + subdirs[:] = [ + name + for name in subdirs + if not any( + fnmatch_ex(pattern, current / name) + for pattern in config.getini("norecursedirs") + ) + and current / name not in ignored + and not any( + fnmatch_ex(pattern, current / name) for pattern in ignored_globs + ) + ] + for name in names: + module_path = current / name + relative_path = _relative_path(module_path, config) + if ( + module_path not in ignored + and not any( + fnmatch_ex(pattern, module_path) for pattern in ignored_globs + ) + and ( + relative_path in cached_modules + or any(fnmatch_ex(pattern, module_path) for pattern in patterns) + ) + ): + yield relative_path + + +def _ignore_collect_path(collection_path: Path, config: "Config") -> "bool | None": + return _ignore_module(collection_path, config) + + +def _ignore_collect_legacy(path: "os.PathLike[str]", config: "Config") -> "bool | None": + return _ignore_module(Path(path), config) + + +# Pytest 7 introduced collection_path; register only the supported signature. +# Assign the hook by name because pytest 5's pluggy lacks hookimpl(specname=...). +pytest_ignore_collect = hookimpl(tryfirst=True)( + _ignore_collect_path + if int(pytest.__version__.split(".", 1)[0]) >= 7 # noqa: PLR2004 + else _ignore_collect_legacy +) + + class Base: def __init__(self, config: "Config") -> None: """ @@ -139,16 +277,44 @@ def __init__(self, config: "Config") -> None: class PytestSplitPlugin(Base): def __init__(self, config: "Config"): super().__init__(config) + self.module_groups: dict[str, int] = {} + self.module_grouping = algorithms.ModuleGrouping( + algorithms.Algorithms[config.option.splitting_algorithm].value + ) if not self.cached_durations: message = self.writer.markup( "\n[pytest-split] No test durations found. Pytest-split will " - "split tests evenly when no durations are found. " + "split tests (or modules with --no-module-splits) evenly when no durations are found. " "\n[pytest-split] You can expect better results in consequent runs, " "when test timings have been documented.\n" ) self.writer.line(message) + @hookimpl(hookwrapper=True) + def pytest_collection(self) -> "Iterator[None]": + if not self.config.option.no_module_splits: + yield + return + + self.module_groups = self.module_grouping.module_groups( + self.config.option.splits, + _module_paths(self.config, self.cached_durations), + self.cached_durations, + ) + # Explicit module/node arguments bypass pytest_ignore_collect. Filter them + # before pytest creates collectors, including when all arguments vanish. + original_args = self.config.args + self.config.args = [ + arg + for arg in original_args + if not _ignore_module(_collection_path(arg, self.config), self.config) + ] + try: + yield + finally: + self.config.args = original_args + @hookimpl(trylast=True) def pytest_collection_modifyitems( self, config: "Config", items: "list[nodes.Item]" @@ -160,10 +326,16 @@ def pytest_collection_modifyitems( group_idx: int = config.option.group algo = algorithms.Algorithms[config.option.splitting_algorithm].value - groups = algo(splits, items, self.cached_durations) + if config.option.no_module_splits: + groups = self.module_grouping( + splits, items, self.cached_durations, module_groups=self.module_groups + ) + else: + groups = algo(splits, items, self.cached_durations) group = groups[group_idx - 1] - ensure_ipynb_compatibility(group, items) + if not config.option.no_module_splits: + ensure_ipynb_compatibility(group, items) items[:] = group.selected config.hook.pytest_deselected(items=group.deselected) diff --git a/tests/test_module_partition.py b/tests/test_module_partition.py new file mode 100644 index 0000000..f349222 --- /dev/null +++ b/tests/test_module_partition.py @@ -0,0 +1,485 @@ +import json +from types import SimpleNamespace +from typing import cast + +import pytest +from _pytest.main import ExitCode # type: ignore[attr-defined] +from pytest_split.algorithms import Algorithms, ModuleGrouping + +pytest_plugins = ["pytester"] + + +def item(nodeid: str) -> pytest.Item: + return cast("pytest.Item", SimpleNamespace(nodeid=nodeid)) + + +class TestModuleGrouping: + @pytest.mark.parametrize( + ("splitting_algorithm", "expected"), + [ + ( + "duration_based_chunks", + {"test_a.py": 0, "test_b.py": 0, "test_c.py": 1, "test_d.py": 1}, + ), + ( + "least_duration", + {"test_a.py": 0, "test_b.py": 1, "test_c.py": 1, "test_d.py": 0}, + ), + ], + ) + def test_module_plan_uses_selected_algorithm(self, splitting_algorithm, expected): + modules = {f"test_{name}.py" for name in "abcd"} + durations = { + f"test_{name}.py::test_one": duration + for name, duration in zip("abcd", [8.0, 7.0, 6.0, 5.0], strict=True) + } + grouping = ModuleGrouping(Algorithms[splitting_algorithm].value) + assert grouping.module_groups(2, modules, durations) == expected + + @pytest.mark.parametrize("splitting_algorithm", Algorithms.names()) + @pytest.mark.parametrize("splits", [1, 2, 6]) + @pytest.mark.parametrize("zero_durations", [False, True]) + def test_module_plan_handles_missing_and_zero_durations( + self, splitting_algorithm, splits, zero_durations + ): + modules = {f"test_{name}.py" for name in "abcd"} + durations = ( + {f"{path}::test_one": 0.0 for path in modules} if zero_durations else {} + ) + grouping = ModuleGrouping(Algorithms[splitting_algorithm].value) + plan = grouping.module_groups(splits, modules, durations) + assert set(plan) == modules + assert all(0 <= group < splits for group in plan.values()) + expected = {1: [0, 0, 0, 0], 2: [0, 0, 1, 1], 6: [0, 1, 2, 3]} + assert sorted(plan.values()) == expected[splits] + + @pytest.mark.parametrize("splitting_algorithm", Algorithms.names()) + def test_module_plan_handles_no_modules(self, splitting_algorithm): + grouping = ModuleGrouping(Algorithms[splitting_algorithm].value) + assert grouping.module_groups(2, set(), {}) == {} + + def test_reuses_full_plan_when_only_one_group_has_been_collected(self): + algorithm = ModuleGrouping(Algorithms["least_duration"].value) + durations: dict[str, float] = { + "test_a.py::test_one": 8, + "test_b.py::test_one": 2, + } + plan = algorithm.module_groups(2, {"test_a.py", "test_b.py"}, durations) + selected = item("test_b.py::test_one") + first, second = algorithm(2, [selected], durations, module_groups=plan) + assert first.selected == [] + assert second.selected == [selected] + + def test_unplanned_modules_are_split_without_changing_the_original_plan(self): + algorithm = ModuleGrouping(Algorithms["least_duration"].value) + plan = {"test_a.py": 1} + items = [item(f"test_{name}.py::test_one") for name in "abc"] + first, second = algorithm(2, items, {}, module_groups=plan) + assert first.selected == [items[1]] + assert second.selected == [items[0], items[2]] + assert plan == {"test_a.py": 1} + + def test_balances_summed_module_durations(self): + durations: dict[str, float] = { + "test_a.py::test_one": 4, + "test_a.py::test_two": 4, + "test_b.py::test_one": 7, + "test_c.py::test_one": 6, + "test_d.py::test_one": 5, + } + items = [item(nodeid) for nodeid in durations] + first, second = ModuleGrouping(Algorithms["least_duration"].value)( + 2, items, durations + ) + assert first.selected == [items[0], items[1], items[4]] + assert second.selected == [items[2], items[3]] + assert (first.duration, second.duration) == (13, 13) + + def test_new_modules_use_average_module_duration_and_stale_entries_are_ignored( + self, + ): + modules = {f"test_{name}.py" for name in "abcd"} + durations: dict[str, float] = { + "test_a.py::test_one": 4, + "test_a.py::test_two": 4, + "test_b.py::test_one": 2, + "test_deleted.py::test_one": 1000, + } + groups = ModuleGrouping(Algorithms["least_duration"].value).module_groups( + 2, modules, durations + ) + assert groups == { + "test_a.py": 0, + "test_b.py": 0, + "test_c.py": 1, + "test_d.py": 1, + } + + @pytest.mark.parametrize("duration", [0, 1]) + def test_ties_are_deterministic_and_spread_modules_evenly(self, duration): + modules = {f"test_{name}.py" for name in "abcd"} + durations: dict[str, float] = { + f"{path}::test_one": duration for path in modules + } + groups = ModuleGrouping(Algorithms["least_duration"].value).module_groups( + 2, modules, durations + ) + assert groups == { + "test_a.py": 0, + "test_b.py": 1, + "test_c.py": 0, + "test_d.py": 1, + } + + def test_groups_whole_modules_and_estimates_durations(self): + items = [ + item("test_a.py::test_first[1]"), + item("test_b.py::TestExample::test_first"), + item("test_a.py::test_first[2]"), + item("test_c.py::test_first"), + ] + durations: dict[str, float] = { + items[0].nodeid: 2, + items[1].nodeid: 4, + items[2].nodeid: 3, + } + + first, second = ModuleGrouping(Algorithms["least_duration"].value)( + 2, items, durations + ) + + assert first.selected == [items[0], items[2]] + assert first.deselected == [items[1], items[3]] + assert (first.duration, second.duration) == (5, 7) + assert second.selected == first.deselected + assert second.deselected == first.selected + + @pytest.mark.parametrize("splits", [1, 2, 3, 10]) + def test_assigns_every_test_once_independent_of_collection_order(self, splits): + items = [item(f"test_{name}.py::test_example") for name in "abcd"] + algorithm = ModuleGrouping(Algorithms["least_duration"].value) + durations: dict[str, float] = {items[0].nodeid: 8, items[1].nodeid: 2} + original = algorithm(splits, items, durations) + reordered = algorithm( + splits, items[::-1], dict(reversed(list(durations.items()))) + ) + + assert sorted(test.nodeid for group in original for test in group.selected) == [ + test.nodeid for test in items + ] + for first, second in zip(original, reordered, strict=True): + assert first.selected == second.selected[::-1] + + def test_keeps_notebook_cells_together(self): + items = [ + item(f"test_{name}.ipynb::Cell {i}") for name in "ab" for i in range(3) + ] + first, second = ModuleGrouping(Algorithms["least_duration"].value)(2, items, {}) + assert first.selected == items[:3] + assert second.selected == items[3:] + + +@pytest.mark.parametrize("splitting_algorithm", Algorithms.names()) +class TestEarlyModuleFiltering: + def test_pyargs_module_paths_are_not_imported_during_planning( + self, testdir, splitting_algorithm + ): + testdir.makepyfile( + test_a="def test_one(): pass", test_b="raise AssertionError('Other group')" + ) + result = testdir.inline_run( + "--splits", + "2", + "--group", + "1", + "--no-module-splits", + "--splitting-algorithm", + splitting_algorithm, + "--pyargs", + "test_a.py", + "test_b.py", + ) + result.assertoutcome(passed=1) + + @pytest.mark.parametrize( + "arguments", [["examplepkg"], ["examplepkg.test_a", "examplepkg.test_b"]] + ) + def test_pyargs_does_not_import_other_groups( + self, testdir, arguments, splitting_algorithm + ): + package = testdir.mkpydir("examplepkg") + package.join("test_a.py").write("def test_one(): pass") + package.join("test_b.py").write("raise AssertionError('Other group')") + testdir.syspathinsert() + result = testdir.inline_run( + "--splits", + "2", + "--group", + "1", + "--no-module-splits", + "--splitting-algorithm", + splitting_algorithm, + "--pyargs", + *arguments, + ) + result.assertoutcome(passed=1) + + def test_norecursedirs_does_not_affect_group_loads( + self, testdir, splitting_algorithm + ): + testdir.makeini("[pytest]\nnorecursedirs = excluded") + testdir.makepyfile( + test_a="def test_one(): pass", test_b="raise AssertionError('Other group')" + ) + excluded = testdir.mkdir("excluded") + excluded.join("test_slow.py").write("raise AssertionError('Ignored directory')") + testdir.tmpdir.join("durations.json").write( + json.dumps( + { + "test_a.py::test_one": 8, + "test_b.py::test_one": 7, + "excluded/test_slow.py::test_one": 1000, + } + ) + ) + result = testdir.inline_run( + "--splits", + "2", + "--group", + "1", + "--no-module-splits", + "--splitting-algorithm", + splitting_algorithm, + "--durations-path", + "durations.json", + ) + result.assertoutcome(passed=1) + + @pytest.mark.parametrize(("group", "selected"), [(1, "ad"), (2, "bc")]) + @pytest.mark.parametrize("legacy_cache", [False, True]) + def test_balances_modules_without_importing_other_groups( # noqa: PLR0913 + self, testdir, group, selected, legacy_cache, splitting_algorithm + ): + if splitting_algorithm == "duration_based_chunks": + selected = "ab" if group == 1 else "cd" + durations = { + "test_a.py::test_one": 4, + "test_a.py::test_two": 4, + "test_b.py::test_one": 7, + "test_c.py::test_one": 6, + "test_d.py::test_one": 5, + } + for name in "abcd": + source = "def test_one(): pass" + if name == "a": + source += "\ndef test_two(): pass" + if name not in selected: + source = "raise AssertionError('Must not import another group')" + testdir.makepyfile(**{f"test_{name}": source}) + cache = list(durations.items()) if legacy_cache else durations + testdir.tmpdir.join("durations.json").write(json.dumps(cache)) + result = testdir.inline_run( + "--splits", + "2", + "--group", + str(group), + "--no-module-splits", + "--splitting-algorithm", + splitting_algorithm, + "--durations-path", + "durations.json", + ) + result.assertoutcome(passed=3 if group == 1 else 2) + passed = result.listoutcomes()[0] + assert {report.nodeid.split(".py", 1)[0] for report in passed} == { + f"test_{name}" for name in selected + } + + @pytest.mark.parametrize("ignore_option", ["--ignore", "--ignore-glob"]) + def test_ignores_excluded_and_deleted_modules_when_balancing( + self, testdir, ignore_option, splitting_algorithm + ): + testdir.makepyfile( + test_a="def test_one(): pass", + test_b="raise AssertionError('Ignored module')", + test_c="raise AssertionError('Other group')", + ) + testdir.tmpdir.join("durations.json").write( + json.dumps( + { + "test_a.py::test_one": 8, + "test_b.py::test_one": 100, + "test_c.py::test_one": 7, + "test_deleted.py::test_one": 1000, + } + ) + ) + result = testdir.inline_run( + "--splits", + "2", + "--group", + "1", + "--no-module-splits", + "--splitting-algorithm", + splitting_algorithm, + "--durations-path", + "durations.json", + ignore_option, + "test_b.py", + ) + result.assertoutcome(passed=1) + + @pytest.mark.parametrize( + ("group", "selected", "excluded"), [(1, "a", "b"), (2, "b", "a")] + ) + @pytest.mark.parametrize( + "arguments", + [ + [], + ["."], + ["test_a.py", "test_b.py"], + ["test_a.py::test_a", "test_b.py::test_b"], + ], + ) + def test_does_not_import_or_collect_other_groups( # noqa: PLR0913 + self, testdir, group, selected, excluded, arguments, splitting_algorithm + ): + testdir.makepyfile( + **{ + f"test_{selected}": f"def test_{selected}(): pass", + f"test_{excluded}": "raise AssertionError('This module must not be imported')", + } + ) + result = testdir.inline_run( + "--splits", + "2", + "--group", + str(group), + "--no-module-splits", + "--splitting-algorithm", + splitting_algorithm, + *arguments, + ) + + result.assertoutcome(passed=1) + collected = [ + call.report.nodeid for call in result.getcalls("pytest_collectreport") + ] + assert f"test_{excluded}.py" not in collected + + @pytest.mark.parametrize("argument", ["test_b.py", "test_b.py::test_b"]) + def test_all_explicit_modules_belong_to_another_group( + self, testdir, argument, splitting_algorithm + ): + testdir.makepyfile( + test_b="raise AssertionError('This module must not be imported')" + ) + result = testdir.inline_run( + "--splits", + "2", + "--group", + "2", + "--no-module-splits", + "--splitting-algorithm", + splitting_algorithm, + argument, + ) + assert result.ret == ExitCode.NO_TESTS_COLLECTED + result.assertoutcome(passed=0) + + def test_recurses_into_directories_and_respects_custom_python_modules( + self, testdir, splitting_algorithm + ): + testdir.makeini("[pytest]\npython_files = check_*.py") + checks = testdir.mkdir("checks") + checks.join("check_a.py").write("def test_selected(): pass") + checks.join("check_b.py").write("raise AssertionError('Must not be imported')") + + result = testdir.inline_run( + "--splits", + "2", + "--group", + "1", + "--no-module-splits", + "--splitting-algorithm", + splitting_algorithm, + ) + result.assertoutcome(passed=1) + + def test_applies_keyword_filtering_to_assigned_modules( + self, testdir, splitting_algorithm + ): + testdir.makepyfile(test_a="def test_keep(): pass\ndef test_drop(): pass") + result = testdir.inline_run( + "--splits", + "2", + "--group", + "1", + "--no-module-splits", + "--splitting-algorithm", + splitting_algorithm, + "-k", + "keep", + ) + result.assertoutcome(passed=1) + deselected = result.getcalls("pytest_deselected") + assert [test.nodeid for call in deselected for test in call.items] == [ + "test_a.py::test_drop" + ] + + def test_no_filtering_without_splits(self, testdir, splitting_algorithm): + testdir.makepyfile(test_a="def test_a(): pass", test_b="def test_b(): pass") + result = testdir.inline_run( + "--no-module-splits", "--splitting-algorithm", splitting_algorithm + ) + result.assertoutcome(passed=2) + + def test_collect_only_skips_importing_other_groups( + self, testdir, splitting_algorithm + ): + testdir.makepyfile( + test_a="def test_a(): pass", + test_b="raise AssertionError('Must not be imported')", + ) + result = testdir.inline_run( + "--splits", + "2", + "--group", + "1", + "--no-module-splits", + "--splitting-algorithm", + splitting_algorithm, + "--collect-only", + ) + assert result.ret == ExitCode.OK + collected = result.getcalls("pytest_itemcollected") + assert [call.item.nodeid for call in collected] == ["test_a.py::test_a"] + + def test_storing_durations_preserves_other_groups( + self, testdir, splitting_algorithm + ): + testdir.makepyfile(test_a="def test_a(): pass", test_b="def test_b(): pass") + durations_path = testdir.tmpdir.join("durations.json") + other_group_duration = 100 + durations_path.write( + json.dumps( + {"test_a.py::test_a": 200, "test_b.py::test_b": other_group_duration} + ) + ) + result = testdir.inline_run( + "--splits", + "2", + "--group", + "1", + "--no-module-splits", + "--splitting-algorithm", + splitting_algorithm, + "--store-durations", + "--durations-path", + str(durations_path), + ) + + result.assertoutcome(passed=1) + durations = json.loads(durations_path.read()) + assert durations["test_b.py::test_b"] == other_group_duration + assert "test_a.py::test_a" in durations