Skip to content

Commit 06bfa69

Browse files
authored
Merge pull request #4870 from Zac-HD/claude/status-reason-ideas-dum8wm
Add status_reason category and location to observability reports
2 parents 1bcbae8 + a0a7dfc commit 06bfa69

19 files changed

Lines changed: 424 additions & 71 deletions

File tree

‎hypothesis/RELEASE.rst‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
RELEASE_TYPE: minor
2+
3+
Test case observations from |observability| now give more detail about why a
4+
test case failed or was abandoned (:issue:`3845`). ``metadata`` includes a new
5+
``status_reason_location`` key: a ``filename:lineno`` location for the
6+
``status_reason``, if known - for example the location of a failing |assume|
7+
call, the |.filter| call whose predicate rejected the last drawn value, or the
8+
exception for failing tests.
9+
10+
Test cases which exceeded the maximum allowed size now also report a nonempty
11+
``status_reason``.

‎hypothesis/docs/reference/schema_metadata.json‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,12 @@
6464
"interesting_origin": {
6565
"type": ["string", "null"],
6666
"description": "The internal ``InterestingOrigin`` object for failing tests, if and only if ``status == \"failed\"``. The ``traceback`` string value is derived from this object."
67+
},
68+
"status_reason_location": {
69+
"type": ["string", "null"],
70+
"description": "A ``filename:lineno`` location for the ``status_reason`` of this test case, if known - for example the location of a failing |assume| call, the |.filter| call whose predicate rejected the last drawn value, or the exception for failing tests."
6771
}
6872
},
69-
"required": ["traceback", "reproduction_decorator", "notes", "predicates", "backend", "sys.argv", "os.getpid()", "imported_at", "data_status", "interesting_origin"],
73+
"required": ["traceback", "reproduction_decorator", "notes", "predicates", "backend", "sys.argv", "os.getpid()", "imported_at", "data_status", "interesting_origin", "status_reason_location"],
7074
"additionalProperties": false
7175
}

‎hypothesis/src/hypothesis/control.py‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,18 +39,24 @@ def _calling_function_location(what: str, frame: Any) -> str:
3939
return f"{what}() in {where.f_code.co_name} (line {where.f_lineno})"
4040

4141

42+
def _calling_frame_location(frame: Any) -> str:
43+
where = frame.f_back
44+
return f"{where.f_code.co_filename}:{where.f_lineno}"
45+
46+
4247
def reject() -> NoReturn:
4348
if _current_build_context.value is None:
4449
note_deprecation(
4550
"Using `reject` outside a property-based test is deprecated",
4651
since="2023-09-25",
4752
has_codemod=False,
4853
)
49-
where = _calling_function_location("reject", inspect.currentframe())
54+
frame = inspect.currentframe()
55+
where = _calling_function_location("reject", frame)
5056
if currently_in_test_context():
5157
counts = current_build_context().data._observability_predicates[where]
5258
counts.update_count(condition=False)
53-
raise UnsatisfiedAssumption(where)
59+
raise UnsatisfiedAssumption(where, location=_calling_frame_location(frame))
5460

5561

5662
@overload
@@ -73,12 +79,16 @@ def assume(condition: object) -> Literal[True]:
7379
has_codemod=False,
7480
)
7581
if observability_enabled() or not condition:
76-
where = _calling_function_location("assume", inspect.currentframe())
82+
frame = inspect.currentframe()
83+
where = _calling_function_location("assume", frame)
7784
if observability_enabled() and currently_in_test_context():
7885
counts = current_build_context().data._observability_predicates[where]
7986
counts.update_count(condition=bool(condition))
8087
if not condition:
81-
raise UnsatisfiedAssumption(f"failed to satisfy {where}")
88+
raise UnsatisfiedAssumption(
89+
f"failed to satisfy {where}",
90+
location=_calling_frame_location(frame),
91+
)
8292
return True
8393

8494

‎hypothesis/src/hypothesis/core.py‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1236,7 +1236,7 @@ def _execute_once_for_engine(self, data: ConjectureData) -> None:
12361236
# An "assume" check failed, so instead we inform the engine that
12371237
# this test run was invalid.
12381238
try:
1239-
data.mark_invalid(e.reason)
1239+
data.mark_invalid(e.reason, location=e.location)
12401240
except FlakyReplay as err:
12411241
# This was unexpected, meaning that the assume was flaky.
12421242
# Report it as such.
@@ -1585,7 +1585,14 @@ def run_engine(self):
15851585
coverage=None, # Not recorded when we're replaying the MFE
15861586
status="passed" if sys.exc_info()[0] else "failed",
15871587
status_reason=str(origin or "unexpected/flaky pass"),
1588-
metadata={"traceback": tb},
1588+
metadata={
1589+
"traceback": tb,
1590+
"status_reason_location": (
1591+
f"{origin.filename}:{origin.lineno}"
1592+
if origin and origin.filename
1593+
else None
1594+
),
1595+
},
15891596
)
15901597
deliver_observation(tc)
15911598

@@ -2349,8 +2356,10 @@ def fuzz_one_input(
23492356
except StopTest:
23502357
status = data.status
23512358
return None
2352-
except UnsatisfiedAssumption:
2359+
except UnsatisfiedAssumption as e:
23532360
status = Status.INVALID
2361+
data.events["gave up because"] = e.reason or ""
2362+
data.invalid_location = e.location
23542363
return None
23552364
except BaseException as e:
23562365
# The engine sets data.interesting_origin in

‎hypothesis/src/hypothesis/errors.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,12 @@ class UnsatisfiedAssumption(HypothesisException):
3434
If you're seeing this error something has gone wrong.
3535
"""
3636

37-
def __init__(self, reason: str | None = None) -> None:
37+
def __init__(
38+
self, reason: str | None = None, *, location: str | None = None
39+
) -> None:
3840
self.reason = reason
41+
# "filename:lineno" of the failing assume() or reject() call, if known
42+
self.location = location
3943

4044

4145
class NoSuchExample(HypothesisException):

‎hypothesis/src/hypothesis/internal/conjecture/data.py‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
import types
1515
import weakref
1616
from collections import defaultdict
17-
from collections.abc import Generator, Hashable, Iterable, Iterator, Sequence
17+
from collections.abc import Callable, Generator, Hashable, Iterable, Iterator, Sequence
1818
from contextlib import contextmanager
1919
from dataclasses import dataclass, field
2020
from enum import IntEnum
@@ -75,6 +75,7 @@
7575
)
7676
from hypothesis.internal.intervalsets import IntervalSet
7777
from hypothesis.internal.observability import PredicateCounts
78+
from hypothesis.internal.reflection import function_location
7879
from hypothesis.reporting import debug_report
7980
from hypothesis.utils.conventions import UniqueIdentifier, not_set
8081
from hypothesis.utils.deprecation import note_deprecation
@@ -756,6 +757,11 @@ def __init__(
756757
self._observability_predicates: defaultdict[str, PredicateCounts] = defaultdict(
757758
PredicateCounts
758759
)
760+
self.invalid_location: str | None = None
761+
# (predicate, location) of the most recent filter rejection
762+
self._last_rejected_filter: tuple[Callable[[Any], Any], str | None] | None = (
763+
None
764+
)
759765

760766
self._sampled_from_all_strategies_elements_message: (
761767
tuple[str, object] | None
@@ -1264,7 +1270,7 @@ def draw(
12641270
strategy.validate()
12651271

12661272
if strategy.is_empty:
1267-
self.mark_invalid(f"empty strategy {self!r}")
1273+
self.mark_invalid(f"empty strategy {strategy!r}")
12681274

12691275
if self.depth >= MAX_DEPTH:
12701276
self.mark_invalid("max depth exceeded")
@@ -1481,14 +1487,25 @@ def conclude_test(
14811487
def mark_interesting(self, interesting_origin: InterestingOrigin) -> NoReturn:
14821488
self.conclude_test(Status.INTERESTING, interesting_origin)
14831489

1484-
def mark_invalid(self, why: str | None = None) -> NoReturn:
1490+
def mark_invalid(
1491+
self, why: str | None = None, *, location: str | None = None
1492+
) -> NoReturn:
14851493
if why is not None:
1486-
self.events["invalid because"] = why
1494+
self.events["gave up because"] = why
1495+
self.invalid_location = location
14871496
self.conclude_test(Status.INVALID)
14881497

14891498
def mark_overrun(self) -> NoReturn:
14901499
self.conclude_test(Status.OVERRUN)
14911500

1501+
def last_rejected_filter_location(self) -> str | None:
1502+
"""The location of the most recently rejected filter."""
1503+
if self._last_rejected_filter is None:
1504+
return None
1505+
condition, location = self._last_rejected_filter
1506+
# fall back to where the predicate was defined if no location is known
1507+
return location or function_location(condition)
1508+
14921509

14931510
def draw_choice(
14941511
choice_type: ChoiceTypeT, constraints: ChoiceConstraintsT, *, random: Random

‎hypothesis/src/hypothesis/internal/conjecture/engine.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -989,7 +989,7 @@ def debug_data(self, data: ConjectureData | ConjectureResult) -> None:
989989
status = f"{status} ({data.interesting_origin!r})"
990990
elif data.status == Status.INVALID and isinstance(data, ConjectureData):
991991
assert isinstance(data, ConjectureData) # mypy is silly
992-
status = f"{status} ({data.events.get('invalid because', '?')})"
992+
status = f"{status} ({data.events.get('gave up because', '?')})"
993993

994994
self.debug(f"{len(data.choices)} choices -> {status}\n\t{data.choices}")
995995

@@ -1361,9 +1361,9 @@ def generate_new_test_cases(self) -> None:
13611361
if (
13621362
data.status is Status.OVERRUN
13631363
and max_length is not None
1364-
and "invalid because" not in data.events
1364+
and "gave up because" not in data.events
13651365
):
1366-
data.events["invalid because"] = (
1366+
data.events["gave up because"] = (
13671367
"reduced max size for early test cases (avoids flaky health checks)"
13681368
)
13691369

‎hypothesis/src/hypothesis/internal/observability.py‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,7 @@ class ObservationMetadata:
182182
data_status: "Status"
183183
phase: str
184184
interesting_origin: InterestingOrigin | None
185+
status_reason_location: str | None
185186
choice_nodes: tuple[ChoiceNode, ...] | None
186187
choice_spans: Optional["Spans"]
187188

@@ -198,6 +199,7 @@ def to_json(self) -> dict[str, Any]:
198199
"data_status": self.data_status,
199200
"phase": self.phase,
200201
"interesting_origin": self.interesting_origin,
202+
"status_reason_location": self.status_reason_location,
201203
"choice_nodes": (
202204
None if self.choice_nodes is None else nodes_to_json(self.choice_nodes)
203205
),
@@ -425,8 +427,13 @@ def make_testcase(
425427
status_reason = str(data.interesting_origin)
426428
elif phase == "shrink" and data.status == Status.OVERRUN:
427429
status_reason = "exceeded size of current best test case"
430+
elif data.status == Status.OVERRUN:
431+
status_reason = (
432+
str(data.events.pop("gave up because", ""))
433+
or "exceeded maximum test case size"
434+
)
428435
else:
429-
status_reason = str(data.events.pop("invalid because", ""))
436+
status_reason = str(data.events.pop("gave up because", ""))
430437

431438
status_map: dict[Status, TestCaseStatus] = {
432439
Status.OVERRUN: "gave_up",
@@ -440,6 +447,13 @@ def make_testcase(
440447
if status is None:
441448
status = status_map[data.status]
442449

450+
status_reason_location: str | None = None
451+
if (origin := data.interesting_origin) is not None:
452+
if origin.filename is not None:
453+
status_reason_location = f"{origin.filename}:{origin.lineno}"
454+
elif status != "failed":
455+
status_reason_location = data.invalid_location
456+
443457
return TestCaseObservation(
444458
type="test_case",
445459
status=status,
@@ -469,6 +483,7 @@ def make_testcase(
469483
"data_status": data.status,
470484
"phase": phase,
471485
"interesting_origin": data.interesting_origin,
486+
"status_reason_location": status_reason_location,
472487
"choice_nodes": data.nodes if OBSERVABILITY_CHOICES else None,
473488
"choice_spans": data.spans if OBSERVABILITY_CHOICES else None,
474489
**_system_metadata(),

‎hypothesis/src/hypothesis/internal/reflection.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,15 @@ def is_first_param_referenced_in_function(f: Any) -> bool:
268268
)
269269

270270

271+
def function_location(f: object) -> str | None:
272+
"""Return a ``filename:lineno`` string for where ``f`` was defined, if known."""
273+
try:
274+
code = f.__code__ # type: ignore[attr-defined]
275+
return f"{code.co_filename}:{code.co_firstlineno}"
276+
except AttributeError:
277+
return None
278+
279+
271280
def get_pretty_function_description(f: object) -> str:
272281
# Anything which knows how to pretty-print itself knows better than we do -
273282
# e.g. the constant functions from st.functions(), which would otherwise

‎hypothesis/src/hypothesis/stateful.py‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
SampledFromStrategy,
5858
SearchStrategy,
5959
check_strategy,
60+
current_filter_call_site,
6061
filter_not_satisfied,
6162
)
6263
from hypothesis.utils.deprecation import note_deprecation
@@ -535,7 +536,7 @@ def __post_init__(self):
535536
# stored on the machine and we want to draw the transformed value
536537
# directly instead of a reference.
537538
draw_references = all(
538-
name == "filter" for name, _ in v._transformations
539+
name == "filter" for name, *_ in v._transformations
539540
)
540541
v = Bundle(
541542
v.name,
@@ -594,13 +595,15 @@ def __init__(self, machine, elements, *, force_repr, transformations):
594595
)
595596
self.machine = machine
596597

597-
def _transform(self, element):
598+
def _transform(self, element, *, data):
598599
# Filter and map transformations apply to the value, which we look up
599600
# lazily so that untested elements cost nothing; the position and
600601
# reference ride along so that Bundle.do_draw can consume the chosen
601602
# element and refer to it by name.
602603
position, reference = element
603-
result = super()._transform(self.machine.names_to_values[reference.name])
604+
result = super()._transform(
605+
self.machine.names_to_values[reference.name], data=data
606+
)
604607
if result is filter_not_satisfied:
605608
return filter_not_satisfied
606609
return (position, reference, result)
@@ -640,7 +643,7 @@ def __init__(
640643
consume: bool = False,
641644
draw_references: bool = False,
642645
transformations: tuple[
643-
tuple[Literal["filter", "map"], Callable[[Ex], Any]], ...
646+
tuple[Literal["filter", "map"], Callable[[Ex], Any], str | None], ...
644647
] = (),
645648
) -> None:
646649
super().__init__()
@@ -666,7 +669,10 @@ def do_draw(self, data):
666669
)
667670
result = sampler.do_filtered_draw(data)
668671
if result is filter_not_satisfied:
669-
data.mark_invalid(f"Aborted test because unable to satisfy {self!r}")
672+
data.mark_invalid(
673+
f"Aborted test because unable to satisfy {self!r}",
674+
location=data.last_rejected_filter_location(),
675+
)
670676
position, reference, value = result
671677
if self.consume:
672678
bundle.pop(position) # pragma: no cover # coverage is flaky here
@@ -677,19 +683,21 @@ def do_draw(self, data):
677683
# apply within a single draw: the generic wrappers instead retry the whole
678684
# draw when a filter fails, consuming rejected values from consuming
679685
# bundles, and are opaque to the reference-drawing logic in Rule.
680-
def __with_transform(self, name, f):
686+
def __with_transform(self, name, f, *, location=None):
681687
return Bundle(
682688
self.name,
683689
consume=self.consume,
684690
draw_references=self.draw_references,
685-
transformations=(*self._transformations, (name, f)),
691+
transformations=(*self._transformations, (name, f, location)),
686692
)
687693

688694
def map(self, pack):
689695
return self.__with_transform("map", pack)
690696

691697
def filter(self, condition):
692-
return self.__with_transform("filter", condition)
698+
return self.__with_transform(
699+
"filter", condition, location=current_filter_call_site()
700+
)
693701

694702
def _base_repr(self):
695703
consume = ", consume=True" if self.consume else ""
@@ -698,7 +706,7 @@ def _base_repr(self):
698706
def __repr__(self):
699707
transforms = "".join(
700708
f".{name}({get_pretty_function_description(f)})"
701-
for name, f in self._transformations
709+
for name, f, _ in self._transformations
702710
)
703711
return self._base_repr() + transforms
704712

0 commit comments

Comments
 (0)