Skip to content

Commit 96fe457

Browse files
art049art049
committed
feat: support pytest-benchmark's pedantic API
1 parent c4adb9b commit 96fe457

File tree

8 files changed

+530
-127
lines changed

8 files changed

+530
-127
lines changed

‎src/pytest_codspeed/config.py‎

Lines changed: 51 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,14 @@
11
from __future__ import annotations
22

3-
from dataclasses import dataclass
4-
from typing import TYPE_CHECKING
3+
import dataclasses
4+
from dataclasses import dataclass, field
5+
from typing import TYPE_CHECKING, Generic, TypeVar
6+
7+
T = TypeVar("T")
58

69
if TYPE_CHECKING:
10+
from typing import Any, Callable
11+
712
import pytest
813

914

@@ -64,17 +69,51 @@ def from_pytest_item(cls, item: pytest.Item) -> BenchmarkMarkerOptions:
6469
raise ValueError(
6570
"Positional arguments are not allowed in the benchmark marker"
6671
)
72+
kwargs = marker.kwargs
6773

68-
options = cls(
69-
group=marker.kwargs.pop("group", None),
70-
min_time=marker.kwargs.pop("min_time", None),
71-
max_time=marker.kwargs.pop("max_time", None),
72-
max_rounds=marker.kwargs.pop("max_rounds", None),
73-
)
74-
75-
if len(marker.kwargs) > 0:
74+
unknown_kwargs = set(kwargs.keys()) - {
75+
field.name for field in dataclasses.fields(cls)
76+
}
77+
if unknown_kwargs:
7678
raise ValueError(
7779
"Unknown kwargs passed to benchmark marker: "
78-
+ ", ".join(marker.kwargs.keys())
80+
+ ", ".join(sorted(unknown_kwargs))
7981
)
80-
return options
82+
83+
return cls(**kwargs)
84+
85+
86+
@dataclass(frozen=True)
87+
class PedanticOptions(Generic[T]):
88+
"""Parameters for running a benchmark using the pedantic fixture API."""
89+
90+
target: Callable[..., T]
91+
setup: Callable[[], Any | None] | None
92+
teardown: Callable[..., Any | None] | None
93+
rounds: int
94+
warmup_rounds: int
95+
iterations: int
96+
args: tuple[Any, ...] = field(default_factory=tuple)
97+
kwargs: dict[str, Any] = field(default_factory=dict)
98+
99+
def __post_init__(self) -> None:
100+
if self.rounds < 0:
101+
raise ValueError("rounds must be positive")
102+
if self.warmup_rounds < 0:
103+
raise ValueError("warmup_rounds must be non-negative")
104+
if self.iterations <= 0:
105+
raise ValueError("iterations must be positive")
106+
if self.iterations > 1 and self.setup is not None:
107+
raise ValueError(
108+
"setup cannot be used with multiple iterations, use multiple rounds"
109+
)
110+
111+
def setup_and_get_args_kwargs(self) -> tuple[tuple[Any, ...], dict[str, Any]]:
112+
if self.setup is None:
113+
return self.args, self.kwargs
114+
maybe_result = self.setup(*self.args, **self.kwargs)
115+
if maybe_result is not None:
116+
if len(self.args) > 0 or len(self.kwargs) > 0:
117+
raise ValueError("setup cannot return a value when args are provided")
118+
return maybe_result
119+
return self.args, self.kwargs

‎src/pytest_codspeed/instruments/__init__.py‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,14 @@
55
from typing import TYPE_CHECKING
66

77
if TYPE_CHECKING:
8-
from typing import Any, Callable, ClassVar, ParamSpec, TypeVar
8+
from typing import Any, Callable, ClassVar, TypeVar
99

1010
import pytest
1111

12-
from pytest_codspeed.config import BenchmarkMarkerOptions
12+
from pytest_codspeed.config import BenchmarkMarkerOptions, PedanticOptions
1313
from pytest_codspeed.plugin import CodSpeedConfig
1414

1515
T = TypeVar("T")
16-
P = ParamSpec("P")
1716

1817

1918
class Instrument(metaclass=ABCMeta):
@@ -31,9 +30,18 @@ def measure(
3130
marker_options: BenchmarkMarkerOptions,
3231
name: str,
3332
uri: str,
34-
fn: Callable[P, T],
35-
*args: P.args,
36-
**kwargs: P.kwargs,
33+
fn: Callable[..., T],
34+
*args: tuple,
35+
**kwargs: dict[str, Any],
36+
) -> T: ...
37+
38+
@abstractmethod
39+
def measure_pedantic(
40+
self,
41+
marker_options: BenchmarkMarkerOptions,
42+
pedantic_options: PedanticOptions[T],
43+
name: str,
44+
uri: str,
3745
) -> T: ...
3846

3947
@abstractmethod

‎src/pytest_codspeed/instruments/valgrind.py‎

Lines changed: 52 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from __future__ import annotations
22

33
import sys
4+
import warnings
45
from typing import TYPE_CHECKING
56

67
from pytest_codspeed import __semver_version__
@@ -12,7 +13,8 @@
1213

1314
from pytest import Session
1415

15-
from pytest_codspeed.instruments import P, T
16+
from pytest_codspeed.config import PedanticOptions
17+
from pytest_codspeed.instruments import T
1618
from pytest_codspeed.plugin import BenchmarkMarkerOptions, CodSpeedConfig
1719

1820
SUPPORTS_PERF_TRAMPOLINE = sys.version_info >= (3, 12)
@@ -52,9 +54,9 @@ def measure(
5254
marker_options: BenchmarkMarkerOptions,
5355
name: str,
5456
uri: str,
55-
fn: Callable[P, T],
56-
*args: P.args,
57-
**kwargs: P.kwargs,
57+
fn: Callable[..., T],
58+
*args: tuple,
59+
**kwargs: dict[str, Any],
5860
) -> T:
5961
self.benchmark_count += 1
6062

@@ -78,8 +80,54 @@ def __codspeed_root_frame__() -> T:
7880
self.instrument_hooks.lib.callgrind_stop_instrumentation()
7981
self.instrument_hooks.set_executed_benchmark(uri)
8082

83+
def measure_pedantic(
84+
self,
85+
marker_options: BenchmarkMarkerOptions,
86+
pedantic_options: PedanticOptions[T],
87+
name: str,
88+
uri: str,
89+
) -> T:
90+
if pedantic_options.rounds != 1 or pedantic_options.iterations != 1:
91+
warnings.warn(
92+
"Valgrind instrument ignores rounds and iterations settings "
93+
"in pedantic mode"
94+
)
95+
if not self.instrument_hooks:
96+
args, kwargs = pedantic_options.setup_and_get_args_kwargs()
97+
out = pedantic_options.target(*args, **kwargs)
98+
if pedantic_options.teardown is not None:
99+
pedantic_options.teardown(*args, **kwargs)
100+
return out
101+
102+
def __codspeed_root_frame__(*args, **kwargs) -> T:
103+
return pedantic_options.target(*args, **kwargs)
104+
105+
# Warmup
106+
warmup_rounds = max(
107+
pedantic_options.warmup_rounds, 1 if SUPPORTS_PERF_TRAMPOLINE else 0
108+
)
109+
for _ in range(warmup_rounds):
110+
args, kwargs = pedantic_options.setup_and_get_args_kwargs()
111+
__codspeed_root_frame__(*args, **kwargs)
112+
if pedantic_options.teardown is not None:
113+
pedantic_options.teardown(*args, **kwargs)
114+
115+
# Compute the actual result of the function
116+
args, kwargs = pedantic_options.setup_and_get_args_kwargs()
117+
self.instrument_hooks.lib.callgrind_start_instrumentation()
118+
try:
119+
out = __codspeed_root_frame__(*args, **kwargs)
120+
finally:
121+
self.instrument_hooks.lib.callgrind_stop_instrumentation()
122+
self.instrument_hooks.set_executed_benchmark(uri)
123+
if pedantic_options.teardown is not None:
124+
pedantic_options.teardown(*args, **kwargs)
125+
126+
return out
127+
81128
def report(self, session: Session) -> None:
82129
reporter = session.config.pluginmanager.get_plugin("terminalreporter")
130+
assert reporter is not None, "terminalreporter not found"
83131
count_suffix = "benchmarked" if self.should_measure else "benchmark tested"
84132
reporter.write_sep(
85133
"=",

0 commit comments

Comments
 (0)