Skip to content

Commit 78c5aae

Browse files
Allow for per-runner configuration of timeout_minutes (#477)
2 parents 8c67b1d + 2d39275 commit 78c5aae

7 files changed

Lines changed: 102 additions & 0 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
## Unreleased
22

3+
### Configurable benchmark timeout per runner
4+
5+
Set `timeout_minutes` in a runner's `bench_runner.toml` configuration to
6+
override its benchmark job timeout.
7+
38
### Bugfixes
49

510
#### Use PGO on weekly builds

‎README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,12 @@ You may limit the number of cores used to build Python with the `use_cores` opti
9898
use_cores = 2
9999
```
100100

101+
Set `timeout_minutes` to a positive integer to limit a runner's entire benchmark job runtime in minutes, including building CPython and running the benchmarks. If omitted, the workflow template's default applies (Linux: 1440; otherwise GitHub's default).
102+
103+
```
104+
timeout_minutes = 210
105+
```
106+
101107
### Try a benchmarking run
102108

103109
There are instructions for running a benchmarking action already in the `README.md` of your repo. Look there and give it a try!

‎bench_runner/runners.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,16 @@ class Runner:
4646
# The number of cores to use to compile CPython. If not provided, `make -j`
4747
# will be used.
4848
use_cores: int | None = None
49+
# Benchmark job timeout in minutes. None uses the workflow template's default.
50+
timeout_minutes: int | None = None
4951

5052
def __post_init__(self):
53+
if self.timeout_minutes is not None and (
54+
type(self.timeout_minutes) is not int or self.timeout_minutes <= 0
55+
):
56+
raise ValueError(
57+
f"Runner {self.nickname!r}: timeout_minutes must be a positive integer"
58+
)
5159
if self.github_runner_name is None:
5260
self.github_runner_name = self.name
5361
if self.plot is None:

‎bench_runner/scripts/install.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,12 @@ def generate__benchmark(src: Any) -> Any:
178178

179179
runner_template["runs-on"].append(runner.github_runner_name)
180180

181+
if runner.timeout_minutes is not None:
182+
# Keep the timeout immediately after runs-on in generated YAML.
183+
runner_template.pop("timeout-minutes", None)
184+
idx = list(runner_template).index("runs-on") + 1
185+
runner_template.insert(idx, "timeout-minutes", runner.timeout_minutes)
186+
181187
machine_clauses = [
182188
f"inputs.machine == '{runner.name}'",
183189
"inputs.machine == '__really_all'",

‎tests/data/bench_runner.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ os = "linux"
77
arch = "x86_64"
88
hostname = "pyperf"
99
plot = { name = "Linux", color = "C0" }
10+
timeout_minutes = 210
1011

1112
[longitudinal_plot]
1213
subplots = [

‎tests/test_config.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22
import socket
33

44

5+
import pytest
6+
7+
8+
from bench_runner import config
59
from bench_runner import runners
610

711

@@ -16,3 +20,32 @@ def test_get_runner_for_hostname(monkeypatch):
1620
assert runner.os == "linux"
1721
assert runner.arch == "x86_64"
1822
assert runner.hostname == "pyperf"
23+
assert runner.timeout_minutes == 210
24+
25+
26+
@pytest.mark.parametrize("timeout_minutes", [None, 1, 1440])
27+
def test_runner_timeout_minutes_valid(timeout_minutes):
28+
runner = runners.Runner(
29+
nickname="x",
30+
os="linux",
31+
arch="x86_64",
32+
hostname="x",
33+
timeout_minutes=timeout_minutes,
34+
)
35+
assert runner.timeout_minutes == timeout_minutes
36+
37+
38+
@pytest.mark.parametrize(
39+
"timeout_minutes", ["0", "-1", "true", "false", "1.5", '"210"']
40+
)
41+
def test_runner_timeout_minutes_invalid(tmp_path, timeout_minutes):
42+
cfgpath = tmp_path / "bench_runner.toml"
43+
contents = (DATA_PATH / "bench_runner.toml").read_text()
44+
cfgpath.write_text(
45+
contents.replace(
46+
"timeout_minutes = 210", f"timeout_minutes = {timeout_minutes}"
47+
)
48+
)
49+
50+
with pytest.raises(ValueError, match="timeout_minutes.*must be a positive integer"):
51+
config.get_config(cfgpath)

‎tests/test_install.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
from pathlib import Path
2+
3+
4+
import pytest
5+
6+
7+
from bench_runner import config
8+
from bench_runner import runners
9+
from bench_runner.scripts import install
10+
11+
12+
DATA_PATH = Path(__file__).parent / "data"
13+
14+
15+
@pytest.mark.parametrize(
16+
"os, default_timeout", [("linux", 1440), ("darwin", None), ("windows", None)]
17+
)
18+
def test_benchmark_timeout_minutes(tmp_path, monkeypatch, os, default_timeout):
19+
cfg = config.get_config(DATA_PATH / "bench_runner.toml")
20+
configured = runners.Runner(
21+
nickname="configured",
22+
os=os,
23+
arch="x86_64",
24+
hostname="configured",
25+
timeout_minutes=210,
26+
)
27+
default = runners.Runner(
28+
nickname="default", os=os, arch="x86_64", hostname="default"
29+
)
30+
monkeypatch.setattr(cfg, "runners", {"configured": configured, "default": default})
31+
monkeypatch.setattr(config, "get_config", lambda: cfg)
32+
33+
source = install.load_yaml(install.TEMPLATE_PATH / "_benchmark.src.yml")
34+
output = tmp_path / "_benchmark.yml"
35+
install.write_yaml(output, install.generate__benchmark(source), check=False)
36+
jobs = install.load_yaml(output)["jobs"]
37+
38+
assert jobs[f"benchmark-{configured.name}"]["timeout-minutes"] == 210
39+
default_job = jobs[f"benchmark-{default.name}"]
40+
if default_timeout is None:
41+
assert "timeout-minutes" not in default_job
42+
else:
43+
assert default_job["timeout-minutes"] == default_timeout

0 commit comments

Comments
 (0)