Skip to content

Commit ab90309

Browse files
Liam-DeVoeLiam-DeVoe
authored
Merge pull request #4416 from tybug/provider-on-observability
Add `PrimitiveProvider.on_observation`
2 parents 4d86adf + fa0d09e commit ab90309

7 files changed

Lines changed: 136 additions & 5 deletions

File tree

‎hypothesis-python/RELEASE.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
RELEASE_TYPE: minor
2+
3+
Add |PrimitiveProvider.on_observation| to the internal :ref:`alternative backends <alternative-backends-internals>` interface.

‎hypothesis-python/docs/prolog.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,9 @@
115115
.. |PrimitiveProvider.draw_float| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.draw_float`
116116
.. |PrimitiveProvider.draw_string| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.draw_string`
117117
.. |PrimitiveProvider.draw_bytes| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.draw_bytes`
118+
.. |PrimitiveProvider.on_observation| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.on_observation`
119+
.. |PrimitiveProvider.per_test_case_context_manager| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.per_test_case_context_manager`
120+
.. |PrimitiveProvider.add_observability_callback| replace:: :data:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.add_observability_callback`
118121

119122
.. |AVAILABLE_PROVIDERS| replace:: :data:`~hypothesis.internal.conjecture.providers.AVAILABLE_PROVIDERS`
120123
.. |TESTCASE_CALLBACKS| replace:: :data:`~hypothesis.internal.observability.TESTCASE_CALLBACKS`

‎hypothesis-python/src/hypothesis/core.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@
6868
Unsatisfiable,
6969
UnsatisfiedAssumption,
7070
)
71+
from hypothesis.internal import observability
7172
from hypothesis.internal.compat import (
7273
PYPY,
7374
BaseExceptionGroup,
@@ -99,7 +100,6 @@
99100
)
100101
from hypothesis.internal.healthcheck import fail_health_check
101102
from hypothesis.internal.observability import (
102-
OBSERVABILITY_COLLECT_COVERAGE,
103103
TESTCASE_CALLBACKS,
104104
InfoObservation,
105105
InfoObservationType,
@@ -936,7 +936,9 @@ def test_identifier(self):
936936
) or get_pretty_function_description(self.wrapped_test)
937937

938938
def _should_trace(self):
939-
_trace_obs = TESTCASE_CALLBACKS and OBSERVABILITY_COLLECT_COVERAGE
939+
# NOTE: we explicitly support monkeypatching this. Keep the namespace
940+
# access intact.
941+
_trace_obs = TESTCASE_CALLBACKS and observability.OBSERVABILITY_COLLECT_COVERAGE
940942
_trace_failure = (
941943
self.failed_normally
942944
and not self.failed_due_to_deadline

‎hypothesis-python/src/hypothesis/internal/conjecture/engine.py‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
import time
1616
from collections import defaultdict
1717
from collections.abc import Generator, Sequence
18-
from contextlib import contextmanager, suppress
18+
from contextlib import AbstractContextManager, contextmanager, nullcontext, suppress
1919
from dataclasses import dataclass, field
2020
from datetime import timedelta
2121
from enum import Enum
@@ -68,6 +68,7 @@
6868
from hypothesis.internal.conjecture.shrinker import Shrinker, ShrinkPredicateT, sort_key
6969
from hypothesis.internal.escalation import InterestingOrigin
7070
from hypothesis.internal.healthcheck import fail_health_check
71+
from hypothesis.internal.observability import Observation, with_observation_callback
7172
from hypothesis.reporting import base_report, report
7273

7374
#: The maximum number of times the shrinker will reduce the complexity of a failing
@@ -830,8 +831,31 @@ def debug_data(self, data: Union[ConjectureData, ConjectureResult]) -> None:
830831
f"{', ' + data.output if data.output else ''}"
831832
)
832833

834+
def observe_for_provider(self) -> AbstractContextManager:
835+
def on_observation(observation: Observation) -> None:
836+
assert observation.type == "test_case"
837+
# because lifetime == "test_function"
838+
assert isinstance(self.provider, PrimitiveProvider)
839+
# only fire if we actually used that provider to generate this observation
840+
if not self._switch_to_hypothesis_provider:
841+
self.provider.on_observation(observation)
842+
843+
if (
844+
self.settings.backend != "hypothesis"
845+
# only for lifetime = "test_function" providers (guaranteed
846+
# by this isinstance check)
847+
and isinstance(self.provider, PrimitiveProvider)
848+
# and the provider opted-in to observations
849+
and self.provider.add_observability_callback
850+
):
851+
return with_observation_callback(on_observation)
852+
return nullcontext()
853+
833854
def run(self) -> None:
834-
with local_settings(self.settings):
855+
with (
856+
local_settings(self.settings),
857+
self.observe_for_provider(),
858+
):
835859
try:
836860
self._run()
837861
except RunIsComplete:

‎hypothesis-python/src/hypothesis/internal/conjecture/providers.py‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
from typing import (
2222
TYPE_CHECKING,
2323
Any,
24+
ClassVar,
2425
Literal,
2526
Optional,
2627
TypedDict,
@@ -62,7 +63,7 @@
6263
next_up,
6364
)
6465
from hypothesis.internal.intervalsets import IntervalSet
65-
from hypothesis.internal.observability import InfoObservationType
66+
from hypothesis.internal.observability import InfoObservationType, TestCaseObservation
6667

6768
if TYPE_CHECKING:
6869
from typing import TypeAlias
@@ -356,6 +357,15 @@ class PrimitiveProvider(abc.ABC):
356357
#: Only set this to ``True`` if it is necessary for your backend.
357358
avoid_realization = False
358359

360+
#: If ``True``, |PrimitiveProvider.on_observation| will be added as a
361+
#: callback to |TESTCASE_CALLBACKS|, enabling observability during the lifetime
362+
#: of this provider. If ``False``, |PrimitiveProvider.on_observation| will
363+
#: never be called by Hypothesis.
364+
#:
365+
#: The opt-in behavior of observability is because enabling observability
366+
#: might increase runtime or memory usage.
367+
add_observability_callback: ClassVar[bool] = False
368+
359369
def __init__(self, conjecturedata: Optional["ConjectureData"], /) -> None:
360370
self._cd = conjecturedata
361371

@@ -544,6 +554,41 @@ def observe_information_messages(
544554
assert lifetime in ("test_case", "test_function")
545555
yield from []
546556

557+
def on_observation(self, observation: TestCaseObservation) -> None: # noqa: B027
558+
"""
559+
Called at the end of each test case which uses this provider, with the same
560+
``observation["type"] == "test_case"`` observation that is passed to
561+
other callbacks in |TESTCASE_CALLBACKS|. This method is not called with
562+
``observation["type"] in {"info", "alert", "error"}`` observations.
563+
564+
.. important::
565+
566+
For |PrimitiveProvider.on_observation| to be called by Hypothesis,
567+
|PrimitiveProvider.add_observability_callback| must be set to ``True``,
568+
569+
|PrimitiveProvider.on_observation| is explicitly opt-in, as enabling
570+
observability might increase runtime or memory usage.
571+
572+
Calls to this method are guaranteed to alternate with calls to
573+
|PrimitiveProvider.per_test_case_context_manager|. For example:
574+
575+
.. code-block:: python
576+
577+
# test function starts
578+
per_test_case_context_manager()
579+
on_observation()
580+
per_test_case_context_manager()
581+
on_observation()
582+
...
583+
# test function ends
584+
585+
Note that |PrimitiveProvider.on_observation| will not be called for test
586+
cases which did not use this provider during generation, for example
587+
during |Phase.reuse| or |Phase.shrink|, or because Hypothesis switched
588+
to the standard Hypothesis backend after this backend raised too many
589+
|BackendCannotProceed| exceptions.
590+
"""
591+
547592
def span_start(self, label: int, /) -> None: # noqa: B027 # non-abstract noop
548593
"""Marks the beginning of a semantically meaningful span of choices.
549594

‎hypothesis-python/src/hypothesis/internal/observability.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
import sys
1616
import time
1717
import warnings
18+
from collections.abc import Generator
19+
from contextlib import contextmanager
1820
from dataclasses import dataclass
1921
from datetime import date, timedelta
2022
from functools import lru_cache
@@ -98,6 +100,17 @@ class TestCaseObservation(BaseObservation):
98100
TESTCASE_CALLBACKS: list[Callable[[Observation], None]] = []
99101

100102

103+
@contextmanager
104+
def with_observation_callback(
105+
callback: Callable[[Observation], None],
106+
) -> Generator[None, None, None]:
107+
TESTCASE_CALLBACKS.append(callback)
108+
try:
109+
yield
110+
finally:
111+
TESTCASE_CALLBACKS.remove(callback)
112+
113+
101114
def deliver_observation(observation: Observation) -> None:
102115
for callback in TESTCASE_CALLBACKS:
103116
callback(observation)

‎hypothesis-python/tests/conjecture/test_alt_backend.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
)
4747
from hypothesis.internal.floats import SIGNALING_NAN
4848
from hypothesis.internal.intervalsets import IntervalSet
49+
from hypothesis.internal.observability import TESTCASE_CALLBACKS, Observation
4950

5051
from tests.common.debug import minimal
5152
from tests.common.utils import (
@@ -718,3 +719,43 @@ def test_replay_choices():
718719
# trivial covering test
719720
provider = TrivialProvider(None)
720721
provider.replay_choices([1])
722+
723+
724+
class ObservationProvider(TrivialProvider):
725+
add_observability_callback = True
726+
727+
def __init__(self, conjecturedata: "ConjectureData", /) -> None:
728+
super().__init__(conjecturedata)
729+
# calls to per_test_case_context_manager and on_observation alternate,
730+
# starting with per_test_case_context_manager
731+
self.expected = "per_test_case_context_manager"
732+
733+
@contextmanager
734+
def per_test_case_context_manager(self):
735+
assert self.expected == "per_test_case_context_manager"
736+
self.expected = "on_observation"
737+
yield
738+
739+
def on_observation(self, observation: Observation) -> None:
740+
assert self.expected == "on_observation"
741+
self.expected = "per_test_case_context_manager"
742+
743+
744+
@temp_register_backend("observation", ObservationProvider)
745+
def test_on_observation_alternates():
746+
@given(st.integers())
747+
@settings(backend="observation")
748+
def f(n):
749+
pass
750+
751+
f()
752+
753+
754+
@temp_register_backend("observation", TrivialProvider)
755+
def test_on_observation_no_override():
756+
@given(st.integers())
757+
@settings(backend="observation")
758+
def f(n):
759+
assert TESTCASE_CALLBACKS == []
760+
761+
f()

0 commit comments

Comments
 (0)