Skip to content

Commit 88eaea9

Browse files
akxionelmc
authored andcommitted
Add --benchmark-precision for adaptive rounds (opt-in)
1 parent 3b83d12 commit 88eaea9

8 files changed

Lines changed: 165 additions & 2 deletions

File tree

‎CHANGELOG.rst‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@
22
Changelog
33
=========
44

5+
Unreleased
6+
----------
7+
8+
* Added ``--benchmark-precision`` and ``--benchmark-confidence``:
9+
instead of a fixed number of rounds, stop once the mean's relative margin of error falls below the given fraction.
10+
511
v5.2.3 (2025-11-09)
612
--------------------
713

‎docs/calibration.rst‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,15 @@ as the resolution of the timer).
1616

1717
By default ``pytest-benchmark`` will try to run your function as many times needed to fit a `10 x TIMER_RESOLUTION`
1818
period. You can fine tune this with the ``--benchmark-min-time`` and ``--benchmark-calibration-precision`` options.
19+
20+
How many rounds?
21+
----------------
22+
23+
By default the number of rounds is decided up front:
24+
as many as fit in ``--benchmark-max-time``, but at least ``--benchmark-min-rounds``.
25+
26+
``--benchmark-precision`` makes the stopping point adaptive.
27+
After each round, the relative margin of error of the mean is computed.
28+
Rounds stop once it falls below the target, so ``--benchmark-precision=0.02`` and ``--benchmark-confidence`` at
29+
its default 99% means "stop once the true mean is within ±2% of the measured mean, with 99% confidence".
30+
The usual bounds still apply: never less than ``--benchmark-min-rounds``, never longer than ``--benchmark-max-time``.

‎docs/usage.rst‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,15 @@ Commandline options
8181
--benchmark-min-rounds NUM
8282
Minimum rounds, even if total time would exceed `--max-
8383
time`. Default: 5
84+
--benchmark-precision FRACTION
85+
Run rounds until the relative margin of error of the
86+
mean (at --benchmark-confidence) is below this fraction,
87+
e.g. 0.02 for ±2%. Bounded by --benchmark-min-rounds and
88+
--benchmark-max-time. Default: disabled (fixed number of
89+
rounds).
90+
--benchmark-confidence FRACTION
91+
Confidence level for --benchmark-precision, e.g. 0.99
92+
for a 99% confidence interval. Default: 0.99
8493
--benchmark-timer FUNC
8594
Timer to use when measuring time. Default:
8695
'time.perf_counter'
@@ -264,6 +273,7 @@ You can set per-test options with the ``benchmark`` marker:
264273
min_time=0.1,
265274
max_time=0.5,
266275
min_rounds=5,
276+
precision=0.02,
267277
timer=time.time,
268278
disable_gc=True,
269279
warmup=False

‎src/pytest_benchmark/fixture.py‎

Lines changed: 44 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import traceback
1212
import typing
1313
from math import ceil
14+
from math import sqrt
1415
from pathlib import Path
1516

1617
from .timers import compute_timer_precision
@@ -69,6 +70,8 @@ def __init__(
6970
min_rounds,
7071
min_time,
7172
max_time,
73+
precision,
74+
confidence,
7275
warmup,
7376
warmup_iterations,
7477
calibration_precision,
@@ -100,6 +103,12 @@ def __init__(
100103
self._min_rounds = min_rounds
101104
self._max_time = float(max_time)
102105
self._min_time = float(min_time)
106+
if precision and confidence:
107+
self._confidence = float(confidence)
108+
self._precision = float(precision)
109+
else:
110+
self._confidence = None
111+
self._precision = None
103112
self._add_stats = add_stats
104113
self._calibration_precision = calibration_precision
105114
self._warmup = warmup and warmup_iterations
@@ -159,6 +168,8 @@ def _make_stats(self, iterations):
159168
'min_rounds': self._min_rounds,
160169
'max_time': self._max_time,
161170
'min_time': self._min_time,
171+
'precision': self._precision,
172+
'confidence': self._confidence,
162173
'warmup': self._warmup,
163174
},
164175
)
@@ -231,8 +242,11 @@ def _raw(self, function_to_benchmark, *args, **kwargs):
231242
for _ in range(warmup_rounds):
232243
runner(loops_range)
233244
with PauseInstrumentation():
234-
for _ in range(rounds):
235-
stats.update(runner(loops_range))
245+
if self._precision is None:
246+
for _ in range(rounds):
247+
stats.update(runner(loops_range))
248+
else:
249+
self._run_until_precise(runner, loops_range, stats, rounds)
236250
self._logger.debug(f' Ran for {format_time(time.time() - run_start)}s.', yellow=True, bold=True)
237251
if self.cprofile_loops is None:
238252
cprofile_loops = loops_range or range(1)
@@ -248,6 +262,34 @@ def _raw(self, function_to_benchmark, *args, **kwargs):
248262
function_result = function_to_benchmark(*args, **kwargs)
249263
return function_result
250264

265+
def _run_until_precise(self, runner, loops_range, stats, max_rounds):
266+
# Two-sided normal quantile (good enough for us).
267+
z = statistics.NormalDist().inv_cdf(0.5 + self._confidence / 2)
268+
precision = self._precision
269+
270+
# Welford's online variance algorithm,
271+
# see https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm
272+
mean = 0.0
273+
m2 = 0.0
274+
275+
for n in range(1, max_rounds + 1):
276+
duration = runner(loops_range)
277+
stats.update(duration)
278+
279+
delta = duration - mean
280+
mean += delta / n
281+
m2 += delta * (duration - mean)
282+
283+
if n >= self._min_rounds and n >= 2 and mean > 0:
284+
stddev = sqrt(m2 / (n - 1))
285+
rel_margin = z * stddev / sqrt(n) / mean
286+
if rel_margin <= precision:
287+
self._logger.debug(
288+
f' Reached precision ±{rel_margin:.2%} (target ±{precision:.2%}) after {n} rounds.',
289+
yellow=True,
290+
)
291+
return
292+
251293
def _raw_pedantic(self, target, args=(), kwargs=None, setup=None, teardown=None, rounds=1, warmup_rounds=0, iterations=1):
252294
if kwargs is None:
253295
kwargs = {}

‎src/pytest_benchmark/plugin.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
from .utils import parse_columns
3030
from .utils import parse_compare_fail
3131
from .utils import parse_cprofile_loops
32+
from .utils import parse_fraction
3233
from .utils import parse_name_format
3334
from .utils import parse_rounds
3435
from .utils import parse_save
@@ -193,6 +194,22 @@ def pytest_addoption(parser):
193194
default=5,
194195
help='Minimum rounds, even if total time would exceed `--max-time`. Default: %(default)r',
195196
)
197+
group.addoption(
198+
'--benchmark-precision',
199+
metavar='FRACTION',
200+
type=parse_fraction,
201+
default=None,
202+
help='Run rounds until the relative margin of error of the mean (at --benchmark-confidence) '
203+
'is below this fraction, e.g. 0.02 for ±2%%. Bounded by --benchmark-min-rounds and '
204+
'--benchmark-max-time. Default: disabled (fixed number of rounds).',
205+
)
206+
group.addoption(
207+
'--benchmark-confidence',
208+
metavar='FRACTION',
209+
type=parse_fraction,
210+
default=0.99,
211+
help='Confidence level for --benchmark-precision, e.g. 0.99 for a 99%% confidence interval. Default: %(default)r',
212+
)
196213
group.addoption(
197214
'--benchmark-timer',
198215
metavar='FUNC',
@@ -500,6 +517,8 @@ def pytest_runtest_setup(item):
500517
'max_time',
501518
'min_rounds',
502519
'min_time',
520+
'precision',
521+
'confidence',
503522
'timer',
504523
'group',
505524
'disable_gc',

‎src/pytest_benchmark/session.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@ def __init__(self, config):
5858
'min_time': SecondsDecimal(config.getoption('benchmark_min_time')),
5959
'min_rounds': config.getoption('benchmark_min_rounds'),
6060
'max_time': SecondsDecimal(config.getoption('benchmark_max_time')),
61+
'precision': config.getoption('benchmark_precision'),
62+
'confidence': config.getoption('benchmark_confidence'),
6163
'timer': load_timer(config.getoption('benchmark_timer')),
6264
'calibration_precision': config.getoption('benchmark_calibration_precision'),
6365
'disable_gc': config.getoption('benchmark_disable_gc'),

‎src/pytest_benchmark/utils.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -371,6 +371,17 @@ def parse_rounds(string):
371371
return value
372372

373373

374+
def parse_fraction(string):
375+
try:
376+
value = float(string)
377+
except ValueError as exc:
378+
raise argparse.ArgumentTypeError(exc) from None
379+
else:
380+
if not 0 < value < 1:
381+
raise argparse.ArgumentTypeError('Value must be a fraction strictly between 0 and 1.')
382+
return value
383+
384+
374385
def parse_seconds(string):
375386
try:
376387
return SecondsDecimal(string).as_string

‎tests/test_precision.py‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
"""
2+
Tests for the ``--benchmark-precision`` stopping rule.
3+
4+
Most tests call ``BenchmarkFixture._run_until_precise`` directly with a fake runner
5+
so the statistics are deterministic.
6+
"""
7+
8+
import pytest
9+
10+
11+
class FakeStats:
12+
def __init__(self):
13+
self.data = []
14+
15+
def update(self, duration):
16+
self.data.append(duration)
17+
18+
19+
def make_runner(values):
20+
it = iter(values)
21+
return lambda loops_range: next(it)
22+
23+
24+
def configure(benchmark, precision, min_rounds=5, confidence=0.99):
25+
benchmark._mode = 'benchmark(...)' # mark the fixture as used since we bypass __call__
26+
benchmark._min_rounds = min_rounds
27+
benchmark._precision = precision
28+
benchmark._confidence = confidence
29+
return benchmark
30+
31+
32+
def test_zero_variance_stops_at_min_rounds(benchmark):
33+
# Identical samples have zero margin of error, so it stops exactly at min_rounds.
34+
configure(benchmark, precision=0.01, min_rounds=5)
35+
stats = FakeStats()
36+
benchmark._run_until_precise(make_runner([1.0] * 100), range(1), stats, 50)
37+
assert len(stats.data) == 5
38+
39+
40+
def test_never_precise_runs_to_cap(benchmark):
41+
# Alternating samples never reach the tight margin, so it runs to the max_rounds cap.
42+
configure(benchmark, precision=0.0001, min_rounds=2)
43+
stats = FakeStats()
44+
benchmark._run_until_precise(make_runner([1.0, 5.0] * 100), range(1), stats, 20)
45+
assert len(stats.data) == 20
46+
47+
48+
def test_stops_early_when_precise_enough(benchmark):
49+
# Small but nonzero variance: stops somewhere between min_rounds and the cap.
50+
configure(benchmark, precision=0.05, min_rounds=3)
51+
stats = FakeStats()
52+
samples = [0.99, 1.01] * 500
53+
benchmark._run_until_precise(make_runner(samples), range(1), stats, 1000)
54+
assert 3 <= len(stats.data) < 1000
55+
56+
57+
@pytest.mark.benchmark(precision=0.05, max_time=2.0, min_rounds=3)
58+
def test_precision_marker_end_to_end(benchmark):
59+
# Full path: marker options through the fixture to the adaptive loop.
60+
benchmark(lambda: sum(range(50)))
61+
assert benchmark.stats.stats.rounds >= 3

0 commit comments

Comments
 (0)