Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased]
### Added
- Add `--no-module-splits` for both splitting algorithms to keep modules together and skip importing and collecting modules assigned to other groups.

### Fixed
- Fix malformed bullet points rendering in GitHub Pages documentation

Expand Down
21 changes: 20 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,33 @@ However, when there are major changes in the suite compared to what's stored in

The splitting algorithm can be controlled with the `--splitting-algorithm` CLI option and defaults to `duration_based_chunks`. For more information about the different algorithms and their tradeoffs, please see the section below.

### Input arguments

| Argument | Description |
|----------|-------------|
| `--store-durations` | Store test durations for future runs. |
| `--durations-path PATH` | Path to the durations file. Defaults to `.test_durations` in the current working directory. |
| `--splits N` | Number of groups to split the tests into. Use with `--group`. |
| `--group N` | Group to run, numbered from 1 to `--splits`. |
| `--splitting-algorithm ALGORITHM` | Choose `duration_based_chunks` (default) or `least_duration`. |
| `--no-module-splits` | Keep each module's tests in the same group and skip importing and collecting modules assigned to other groups. Works with either splitting algorithm. |
| `--clean-durations` | Remove stored timings for tests absent from the current run when used with `--store-durations`. |

With `--no-module-splits`, durations are aggregated per module; modules without timings use the average known module duration.
Group assignment happens before collection, using sorted module paths, so test-order randomization does not change which group owns each module.

```sh
pytest --splits 3 --group 1 --no-module-splits --splitting-algorithm least_duration
```

### CLI commands
#### slowest-tests
Lists the slowest tests based on the information stored in the test durations file. See `slowest-tests --help` for more
information.

## Interactions with other pytest plugins
* [`pytest-random-order`](https://github.com/jbasko/pytest-random-order) and [`pytest-randomly`](https://github.com/pytest-dev/pytest-randomly):
⚠️ `pytest-split` running with the `duration_based_chunks` algorithm is **incompatible** with test-order-randomization plugins.
⚠️ `pytest-split` running with the `duration_based_chunks` algorithm without `--no-module-splits` is **incompatible** with test-order-randomization plugins.
Test selection in the groups happens after randomization, potentially causing some tests to be selected in several groups and others not at all.
Instead, a global random seed needs to be computed before running the tests (for example using `$RANDOM` from the shell) and that single seed then needs to be used for all groups by setting the `--random-order-seed` option.

Expand Down
74 changes: 73 additions & 1 deletion src/pytest_split/algorithms.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import heapq
from abc import ABC, abstractmethod
from operator import itemgetter
from typing import TYPE_CHECKING, NamedTuple
from typing import TYPE_CHECKING, NamedTuple, cast

if TYPE_CHECKING:
from _pytest import nodes
Expand Down Expand Up @@ -148,6 +148,78 @@ def __call__(
]


class _Module(NamedTuple):
nodeid: str

def __str__(self) -> str:
return self.nodeid


class ModuleGrouping:
"""Apply a splitting algorithm to whole modules before test collection."""

def __init__(self, algorithm: AlgorithmBase) -> None:
self.algorithm = algorithm

def module_groups(
self, splits: int, modules: "set[str]", durations: "dict[str, float]"
) -> "dict[str, int]":
module_durations: dict[str, float] = {}
for nodeid, duration in durations.items():
path = nodeid.split("::", 1)[0]
if path in modules:
module_durations[path] = module_durations.get(path, 0) + duration

# Both algorithms only read nodeids and string representations. Lightweight
# module representatives let them run without importing test modules.
representatives = cast(
"list[nodes.Item]", [_Module(path) for path in sorted(modules)]
)
if not any(module_durations.values()):
# Equal positive weights also handle cached zero-duration modules.
module_durations = dict.fromkeys(modules, 1.0)
groups = self.algorithm(splits, representatives, module_durations)
return {
module.nodeid: group_idx
for group_idx, group in enumerate(groups)
for module in group.selected
}

def __call__(
self,
splits: int,
items: "list[nodes.Item]",
durations: "dict[str, float]",
*,
module_groups: "dict[str, int] | None" = None,
) -> "list[TestGroup]":
modules = {item.nodeid.split("::", 1)[0] for item in items}
if module_groups is None:
module_groups = self.module_groups(splits, modules, durations)
else:
# Preserve the pre-collection plan. Modules generated by custom
# collectors are collected on every runner and split separately.
unplanned = self.module_groups(
splits, modules - module_groups.keys(), durations
)
module_groups = {**module_groups, **unplanned}
selected: list[list[nodes.Item]] = [[] for _ in range(splits)]
deselected: list[list[nodes.Item]] = [[] for _ in range(splits)]
duration: list[float] = [0 for _ in range(splits)]

for item, item_duration in _get_items_with_durations(items, durations):
group_idx = module_groups[item.nodeid.split("::", 1)[0]]
selected[group_idx].append(item)
duration[group_idx] += item_duration
for i in range(splits):
if i != group_idx:
deselected[i].append(item)

return [
TestGroup(selected[i], deselected[i], duration[i]) for i in range(splits)
]


def _get_items_with_durations(
items: "list[nodes.Item]", durations: "dict[str, float]"
) -> "list[tuple[nodes.Item, float]]":
Expand Down
178 changes: 175 additions & 3 deletions src/pytest_split/plugin.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,20 @@
import importlib.util
import json
import os
from pathlib import Path
from typing import TYPE_CHECKING

import pytest
from _pytest.config import create_terminal_writer, hookimpl
from _pytest.pathlib import fnmatch_ex
from _pytest.reports import TestReport

from pytest_split import algorithms
from pytest_split.ipynb_compatibility import ensure_ipynb_compatibility

if TYPE_CHECKING:
from collections.abc import Iterator

from _pytest import nodes
from _pytest.config import Config
from _pytest.config.argparsing import Parser
Expand Down Expand Up @@ -63,6 +68,14 @@ def pytest_addoption(parser: "Parser") -> None:
default="duration_based_chunks",
choices=algorithms.Algorithms.names(),
)
group.addoption(
"--no-module-splits",
action="store_true",
help=(
"Keep each module's tests in the same group and skip importing and "
"collecting modules assigned to other groups."
),
)
group.addoption(
"--clean-durations",
dest="clean_durations",
Expand Down Expand Up @@ -113,6 +126,131 @@ def pytest_configure(config: "Config") -> None:
)


def _ignore_module(path: Path, config: "Config") -> "bool | None":
if (
config.option.no_module_splits
and config.option.splits
and config.option.group
and path.is_file()
):
plugin = config.pluginmanager.get_plugin("pytestsplitplugin")
if not isinstance(plugin, PytestSplitPlugin):
return None
group_idx = plugin.module_groups.get(_relative_path(path, config))
if group_idx is not None and group_idx != config.option.group - 1:
return True
return None


def _relative_path(path: Path, config: "Config") -> str:
root_path = getattr(config, "rootpath", None)
if root_path is None: # pytest < 6.1
root_path = Path(str(getattr(config, "rootdir"))) # noqa: B009
return Path(os.path.relpath(path, root_path)).as_posix()


def _collection_path(argument: str, config: "Config") -> Path:
name = argument.split("::", 1)[0]
path = config.invocation_params.dir / name
if path.exists():
return path
if config.option.pyargs:
try:
spec = importlib.util.find_spec(name)
except (ImportError, ValueError):
spec = None
if spec is not None:
if spec.submodule_search_locations:
return Path(next(iter(spec.submodule_search_locations)))
if spec.origin is not None:
return Path(spec.origin)
return path


def _module_paths(config: "Config", durations: "dict[str, float]") -> "set[str]":
"""Discover candidate paths without importing test modules."""
patterns = [*config.getini("python_files"), "*.ipynb"]
patterns.extend(config.getoption("doctestglob", default=[]) or ["test*.txt"])
if config.getoption("doctestmodules", default=False):
patterns.append("*.py")
cached_modules = {nodeid.split("::", 1)[0] for nodeid in durations}
modules: set[str] = set()
invocation_dir = config.invocation_params.dir
ignored = [
invocation_dir / path for path in config.getoption("ignore", default=[]) or []
]
ignored_globs = [
str(invocation_dir / pattern)
for pattern in config.getoption("ignore_glob", default=[]) or []
]
for arg in config.args:
path = _collection_path(arg, config)
if path.is_file():
modules.add(_relative_path(path, config))
elif path.is_dir():
modules.update(
_directory_modules(
path, config, patterns, cached_modules, ignored, ignored_globs
)
)
return modules


def _directory_modules( # noqa: PLR0913
path: Path,
config: "Config",
patterns: "list[str]",
cached_modules: "set[str]",
ignored: "list[Path]",
ignored_globs: "list[str]",
) -> "Iterator[str]":
for directory, subdirs, names in os.walk(path):
current = Path(directory)
subdirs[:] = [
name
for name in subdirs
if not any(
fnmatch_ex(pattern, current / name)
for pattern in config.getini("norecursedirs")
)
and current / name not in ignored
and not any(
fnmatch_ex(pattern, current / name) for pattern in ignored_globs
)
]
for name in names:
module_path = current / name
relative_path = _relative_path(module_path, config)
if (
module_path not in ignored
and not any(
fnmatch_ex(pattern, module_path) for pattern in ignored_globs
)
and (
relative_path in cached_modules
or any(fnmatch_ex(pattern, module_path) for pattern in patterns)
)
):
yield relative_path


def _ignore_collect_path(collection_path: Path, config: "Config") -> "bool | None":
return _ignore_module(collection_path, config)


def _ignore_collect_legacy(path: "os.PathLike[str]", config: "Config") -> "bool | None":
return _ignore_module(Path(path), config)


# Pytest 7 introduced collection_path; register only the supported signature.
# Assign the hook by name because pytest 5's pluggy lacks hookimpl(specname=...).
pytest_ignore_collect = hookimpl(tryfirst=True)(
_ignore_collect_path
if int(pytest.__version__.split(".", 1)[0]) >= 7 # noqa: PLR2004
else _ignore_collect_legacy
)


class Base:
def __init__(self, config: "Config") -> None:
"""
Expand All @@ -139,16 +277,44 @@ def __init__(self, config: "Config") -> None:
class PytestSplitPlugin(Base):
def __init__(self, config: "Config"):
super().__init__(config)
self.module_groups: dict[str, int] = {}
self.module_grouping = algorithms.ModuleGrouping(
algorithms.Algorithms[config.option.splitting_algorithm].value
)

if not self.cached_durations:
message = self.writer.markup(
"\n[pytest-split] No test durations found. Pytest-split will "
"split tests evenly when no durations are found. "
"split tests (or modules with --no-module-splits) evenly when no durations are found. "
"\n[pytest-split] You can expect better results in consequent runs, "
"when test timings have been documented.\n"
)
self.writer.line(message)

@hookimpl(hookwrapper=True)
def pytest_collection(self) -> "Iterator[None]":
if not self.config.option.no_module_splits:
yield
return

self.module_groups = self.module_grouping.module_groups(
self.config.option.splits,
_module_paths(self.config, self.cached_durations),
self.cached_durations,
)
# Explicit module/node arguments bypass pytest_ignore_collect. Filter them
# before pytest creates collectors, including when all arguments vanish.
original_args = self.config.args
self.config.args = [
arg
for arg in original_args
if not _ignore_module(_collection_path(arg, self.config), self.config)
]
try:
yield
finally:
self.config.args = original_args

@hookimpl(trylast=True)
def pytest_collection_modifyitems(
self, config: "Config", items: "list[nodes.Item]"
Expand All @@ -160,10 +326,16 @@ def pytest_collection_modifyitems(
group_idx: int = config.option.group

algo = algorithms.Algorithms[config.option.splitting_algorithm].value
groups = algo(splits, items, self.cached_durations)
if config.option.no_module_splits:
groups = self.module_grouping(
splits, items, self.cached_durations, module_groups=self.module_groups
)
else:
groups = algo(splits, items, self.cached_durations)
group = groups[group_idx - 1]

ensure_ipynb_compatibility(group, items)
if not config.option.no_module_splits:
ensure_ipynb_compatibility(group, items)

items[:] = group.selected
config.hook.pytest_deselected(items=group.deselected)
Expand Down
Loading