Skip to content

Commit eca4cff

Browse files
Zac-HDZac-HD
authored
Merge pull request #3159 from Zac-HD/defer-plugin-imports
2 parents 4c64ee5 + 616b2f3 commit eca4cff

File tree

9 files changed

+369
-272
lines changed

9 files changed

+369
-272
lines changed

‎hypothesis-python/.coveragerc‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
[run]
22
branch = True
33
omit =
4-
**/extra/pytestplugin.py
4+
**/_hypothesis_pytestplugin.py
55
**/extra/array_api.py
66
**/extra/cli.py
77
**/extra/django/*.py
88
**/extra/ghostwriter.py
9+
**/extra/pytestplugin.py
910
**/internal/scrutineer.py
1011
**/utils/terminal.py
1112

‎hypothesis-python/RELEASE.rst‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
RELEASE_TYPE: minor
2+
3+
This release modifies our :pypi:`pytest` plugin, to avoid importing Hypothesis
4+
and therefore triggering :ref:`Hypothesis' entry points <entry-points>` for
5+
test suites where Hypothesis is installed but not actually used (:issue:`3140`).

‎hypothesis-python/docs/strategies.rst‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -128,13 +128,14 @@ test is using Hypothesis:
128128
.. _entry-points:
129129

130130
--------------------------------------------------
131-
Registering strategies via setuptools entry points
131+
Hypothesis integration via setuptools entry points
132132
--------------------------------------------------
133133

134134
If you would like to ship Hypothesis strategies for a custom type - either as
135135
part of the upstream library, or as a third-party extension, there's a catch:
136136
:func:`~hypothesis.strategies.from_type` only works after the corresponding
137-
call to :func:`~hypothesis.strategies.register_type_strategy`. This means that
137+
call to :func:`~hypothesis.strategies.register_type_strategy`, and you'll have
138+
the same problem with :func:`~hypothesis.register_random`. This means that
138139
either
139140

140141
- you have to try importing Hypothesis to register the strategy when *your*

‎hypothesis-python/setup.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,8 +129,9 @@ def local_file(name):
129129
"Topic :: Software Development :: Testing",
130130
"Typing :: Typed",
131131
],
132+
py_modules=["_hypothesis_pytestplugin"],
132133
entry_points={
133-
"pytest11": ["hypothesispytest = hypothesis.extra.pytestplugin"],
134+
"pytest11": ["hypothesispytest = _hypothesis_pytestplugin"],
134135
"console_scripts": ["hypothesis = hypothesis.extra.cli:main"],
135136
},
136137
long_description=open(README).read(),

‎hypothesis-python/src/_hypothesis_pytestplugin.py‎

Lines changed: 335 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,335 @@
1+
# This file is part of Hypothesis, which may be found at
2+
# https://github.com/HypothesisWorks/hypothesis/
3+
#
4+
# Most of this work is copyright (C) 2013-2021 David R. MacIver
5+
# ([email protected]), but it contains contributions by others. See
6+
# CONTRIBUTING.rst for a full list of people who may hold copyright, and
7+
# consult the git log if you need to determine who owns an individual
8+
# contribution.
9+
#
10+
# This Source Code Form is subject to the terms of the Mozilla Public License,
11+
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
12+
# obtain one at https://mozilla.org/MPL/2.0/.
13+
#
14+
# END HEADER
15+
16+
"""
17+
The pytest plugin for Hypothesis.
18+
19+
We move this from the old location at `hypothesis.extra.pytestplugin` so that it
20+
can be loaded by Pytest without importing Hypothesis. In turn, this means that
21+
Hypothesis will not load our own third-party plugins (with associated side-effects)
22+
unless and until the user explicitly runs `import hypothesis`.
23+
24+
See https://github.com/HypothesisWorks/hypothesis/issues/3140 for details.
25+
"""
26+
27+
import base64
28+
import sys
29+
from inspect import signature
30+
31+
import pytest
32+
33+
LOAD_PROFILE_OPTION = "--hypothesis-profile"
34+
VERBOSITY_OPTION = "--hypothesis-verbosity"
35+
PRINT_STATISTICS_OPTION = "--hypothesis-show-statistics"
36+
SEED_OPTION = "--hypothesis-seed"
37+
EXPLAIN_OPTION = "--hypothesis-explain"
38+
39+
_VERBOSITY_NAMES = ["quiet", "normal", "verbose", "debug"]
40+
_ALL_OPTIONS = [
41+
LOAD_PROFILE_OPTION,
42+
VERBOSITY_OPTION,
43+
PRINT_STATISTICS_OPTION,
44+
SEED_OPTION,
45+
EXPLAIN_OPTION,
46+
]
47+
_FIXTURE_MSG = """Function-scoped fixture {0!r} used by {1!r}
48+
49+
Function-scoped fixtures are not reset between examples generated by
50+
`@given(...)`, which is often surprising and can cause subtle test bugs.
51+
52+
If you were expecting the fixture to run separately for each generated example,
53+
then unfortunately you will need to find a different way to achieve your goal
54+
(e.g. using a similar context manager instead of a fixture).
55+
56+
If you are confident that your test will work correctly even though the
57+
fixture is not reset between generated examples, you can suppress this health
58+
check to assure Hypothesis that you understand what you are doing.
59+
"""
60+
61+
62+
class StoringReporter:
63+
def __init__(self, config):
64+
assert "hypothesis" in sys.modules
65+
from hypothesis.reporting import default
66+
67+
self.report = default
68+
self.config = config
69+
self.results = []
70+
71+
def __call__(self, msg):
72+
if self.config.getoption("capture", "fd") == "no":
73+
self.report(msg)
74+
if not isinstance(msg, str):
75+
msg = repr(msg)
76+
self.results.append(msg)
77+
78+
79+
# Avoiding distutils.version.LooseVersion due to
80+
# https://github.com/HypothesisWorks/hypothesis/issues/2490
81+
if tuple(map(int, pytest.__version__.split(".")[:2])) < (4, 6): # pragma: no cover
82+
import warnings
83+
84+
PYTEST_TOO_OLD_MESSAGE = """
85+
You are using pytest version %s. Hypothesis tests work with any test
86+
runner, but our pytest plugin requires pytest 4.6 or newer.
87+
Note that the pytest developers no longer support your version either!
88+
Disabling the Hypothesis pytest plugin...
89+
"""
90+
warnings.warn(PYTEST_TOO_OLD_MESSAGE % (pytest.__version__,))
91+
92+
else:
93+
94+
def pytest_addoption(parser):
95+
group = parser.getgroup("hypothesis", "Hypothesis")
96+
group.addoption(
97+
LOAD_PROFILE_OPTION,
98+
action="store",
99+
help="Load in a registered hypothesis.settings profile",
100+
)
101+
group.addoption(
102+
VERBOSITY_OPTION,
103+
action="store",
104+
choices=_VERBOSITY_NAMES,
105+
help="Override profile with verbosity setting specified",
106+
)
107+
group.addoption(
108+
PRINT_STATISTICS_OPTION,
109+
action="store_true",
110+
help="Configure when statistics are printed",
111+
default=False,
112+
)
113+
group.addoption(
114+
SEED_OPTION,
115+
action="store",
116+
help="Set a seed to use for all Hypothesis tests",
117+
)
118+
group.addoption(
119+
EXPLAIN_OPTION,
120+
action="store_true",
121+
help="Enable the `explain` phase for failing Hypothesis tests",
122+
default=False,
123+
)
124+
125+
def _any_hypothesis_option(config):
126+
return bool(any(config.getoption(opt) for opt in _ALL_OPTIONS))
127+
128+
def pytest_report_header(config):
129+
if not (
130+
config.option.verbose >= 1
131+
or "hypothesis" in sys.modules
132+
or _any_hypothesis_option(config)
133+
):
134+
return None
135+
136+
from hypothesis import Verbosity, settings
137+
138+
if config.option.verbose < 1 and settings.default.verbosity < Verbosity.verbose:
139+
return None
140+
settings_str = settings.default.show_changed()
141+
if settings_str != "":
142+
settings_str = f" -> {settings_str}"
143+
return f"hypothesis profile {settings._current_profile!r}{settings_str}"
144+
145+
def pytest_configure(config):
146+
config.addinivalue_line("markers", "hypothesis: Tests which use hypothesis.")
147+
if not _any_hypothesis_option(config):
148+
return
149+
from hypothesis import Phase, Verbosity, core, settings
150+
151+
profile = config.getoption(LOAD_PROFILE_OPTION)
152+
if profile:
153+
settings.load_profile(profile)
154+
verbosity_name = config.getoption(VERBOSITY_OPTION)
155+
if verbosity_name and verbosity_name != settings.default.verbosity.name:
156+
verbosity_value = Verbosity[verbosity_name]
157+
name = f"{settings._current_profile}-with-{verbosity_name}-verbosity"
158+
# register_profile creates a new profile, exactly like the current one,
159+
# with the extra values given (in this case 'verbosity')
160+
settings.register_profile(name, verbosity=verbosity_value)
161+
settings.load_profile(name)
162+
if (
163+
config.getoption(EXPLAIN_OPTION)
164+
and Phase.explain not in settings.default.phases
165+
):
166+
name = f"{settings._current_profile}-with-explain-phase"
167+
phases = settings.default.phases + (Phase.explain,)
168+
settings.register_profile(name, phases=phases)
169+
settings.load_profile(name)
170+
171+
seed = config.getoption(SEED_OPTION)
172+
if seed is not None:
173+
try:
174+
seed = int(seed)
175+
except ValueError:
176+
pass
177+
core.global_force_seed = seed
178+
179+
@pytest.hookimpl(hookwrapper=True)
180+
def pytest_runtest_call(item):
181+
if not (hasattr(item, "obj") and "hypothesis" in sys.modules):
182+
yield
183+
return
184+
185+
from hypothesis import core
186+
from hypothesis.internal.detection import is_hypothesis_test
187+
188+
core.running_under_pytest = True
189+
190+
if not is_hypothesis_test(item.obj):
191+
# If @given was not applied, check whether other hypothesis
192+
# decorators were applied, and raise an error if they were.
193+
if getattr(item.obj, "is_hypothesis_strategy_function", False):
194+
from hypothesis.errors import InvalidArgument
195+
196+
raise InvalidArgument(
197+
f"{item.nodeid} is a function that returns a Hypothesis strategy, "
198+
"but pytest has collected it as a test function. This is useless "
199+
"as the function body will never be executed. To define a test "
200+
"function, use @given instead of @composite."
201+
)
202+
message = "Using `@%s` on a test without `@given` is completely pointless."
203+
for name, attribute in [
204+
("example", "hypothesis_explicit_examples"),
205+
("seed", "_hypothesis_internal_use_seed"),
206+
("settings", "_hypothesis_internal_settings_applied"),
207+
("reproduce_example", "_hypothesis_internal_use_reproduce_failure"),
208+
]:
209+
if hasattr(item.obj, attribute):
210+
from hypothesis.errors import InvalidArgument
211+
212+
raise InvalidArgument(message % (name,))
213+
yield
214+
else:
215+
from hypothesis import HealthCheck, settings
216+
from hypothesis.internal.escalation import current_pytest_item
217+
from hypothesis.internal.healthcheck import fail_health_check
218+
from hypothesis.reporting import with_reporter
219+
from hypothesis.statistics import collector, describe_statistics
220+
221+
# Retrieve the settings for this test from the test object, which
222+
# is normally a Hypothesis wrapped_test wrapper. If this doesn't
223+
# work, the test object is probably something weird
224+
# (e.g a stateful test wrapper), so we skip the function-scoped
225+
# fixture check.
226+
settings = getattr(item.obj, "_hypothesis_internal_use_settings", None)
227+
228+
# Check for suspicious use of function-scoped fixtures, but only
229+
# if the corresponding health check is not suppressed.
230+
if (
231+
settings is not None
232+
and HealthCheck.function_scoped_fixture
233+
not in settings.suppress_health_check
234+
):
235+
# Warn about function-scoped fixtures, excluding autouse fixtures because
236+
# the advice is probably not actionable and the status quo seems OK...
237+
# See https://github.com/HypothesisWorks/hypothesis/issues/377 for detail.
238+
argnames = None
239+
for fx_defs in item._request._fixturemanager.getfixtureinfo(
240+
node=item, func=item.function, cls=None
241+
).name2fixturedefs.values():
242+
if argnames is None:
243+
argnames = frozenset(signature(item.function).parameters)
244+
for fx in fx_defs:
245+
if fx.argname in argnames:
246+
active_fx = item._request._get_active_fixturedef(fx.argname)
247+
if active_fx.scope == "function":
248+
fail_health_check(
249+
settings,
250+
_FIXTURE_MSG.format(fx.argname, item.nodeid),
251+
HealthCheck.function_scoped_fixture,
252+
)
253+
254+
if item.get_closest_marker("parametrize") is not None:
255+
# Give every parametrized test invocation a unique database key
256+
key = item.nodeid.encode()
257+
item.obj.hypothesis.inner_test._hypothesis_internal_add_digest = key
258+
259+
store = StoringReporter(item.config)
260+
261+
def note_statistics(stats):
262+
stats["nodeid"] = item.nodeid
263+
item.hypothesis_statistics = base64.b64encode(
264+
describe_statistics(stats).encode()
265+
).decode()
266+
267+
with collector.with_value(note_statistics):
268+
with with_reporter(store):
269+
with current_pytest_item.with_value(item):
270+
yield
271+
if store.results:
272+
item.hypothesis_report_information = list(store.results)
273+
274+
@pytest.hookimpl(hookwrapper=True)
275+
def pytest_runtest_makereport(item, call):
276+
report = (yield).get_result()
277+
if hasattr(item, "hypothesis_report_information"):
278+
report.sections.append(
279+
("Hypothesis", "\n".join(item.hypothesis_report_information))
280+
)
281+
if hasattr(item, "hypothesis_statistics") and report.when == "teardown":
282+
name = "hypothesis-statistics-" + item.nodeid
283+
try:
284+
item.config._xml.add_global_property(name, item.hypothesis_statistics)
285+
except AttributeError:
286+
# --junitxml not passed, or Pytest 4.5 (before add_global_property)
287+
# We'll fail xunit2 xml schema checks, upgrade pytest if you care.
288+
report.user_properties.append((name, item.hypothesis_statistics))
289+
# If there's an HTML report, include our summary stats for each test
290+
stats = base64.b64decode(item.hypothesis_statistics.encode()).decode()
291+
pytest_html = item.config.pluginmanager.getplugin("html")
292+
if pytest_html is not None: # pragma: no cover
293+
report.extra = getattr(report, "extra", []) + [
294+
pytest_html.extras.text(stats, name="Hypothesis stats")
295+
]
296+
297+
def pytest_terminal_summary(terminalreporter):
298+
if not terminalreporter.config.getoption(PRINT_STATISTICS_OPTION):
299+
return
300+
terminalreporter.section("Hypothesis Statistics")
301+
302+
def report(properties):
303+
for name, value in properties:
304+
if name.startswith("hypothesis-statistics-"):
305+
if hasattr(value, "uniobj"):
306+
# Under old versions of pytest, `value` was a `py.xml.raw`
307+
# rather than a string, so we get the (unicode) string off it.
308+
value = value.uniobj
309+
line = base64.b64decode(value.encode()).decode() + "\n\n"
310+
terminalreporter.write_line(line)
311+
312+
try:
313+
global_properties = terminalreporter.config._xml.global_properties
314+
except AttributeError:
315+
# terminalreporter.stats is a dict, where the empty string appears to
316+
# always be the key for a list of _pytest.reports.TestReport objects
317+
for test_report in terminalreporter.stats.get("", []):
318+
if test_report.when == "teardown":
319+
report(test_report.user_properties)
320+
else:
321+
report(global_properties)
322+
323+
def pytest_collection_modifyitems(items):
324+
if "hypothesis" not in sys.modules:
325+
return
326+
327+
from hypothesis.internal.detection import is_hypothesis_test
328+
329+
for item in items:
330+
if isinstance(item, pytest.Function) and is_hypothesis_test(item.obj):
331+
item.add_marker("hypothesis")
332+
333+
334+
def load():
335+
"""Required for `pluggy` to load a plugin from setuptools entrypoints."""

0 commit comments

Comments
 (0)