@@ -336,6 +336,14 @@ def __init__(
336336 )
337337 self .best_examples_of_observed_targets : dict [str , ConjectureResult ] = {}
338338
339+ # Scheduling state for the target phase. For large max_examples we
340+ # interleave repeated optimisation passes with generation, aiming to
341+ # spend up to half the budget on optimisation in total - see
342+ # _should_optimise_now.
343+ self ._target_valid_spent : int = 0
344+ self ._next_optimise_at : float = math .inf
345+ self ._best_scores_at_last_pass : dict [str , float ] | None = None
346+
339347 # We keep the pareto front in the example database if we have one. This
340348 # is only marginally useful at present, but speeds up local development
341349 # because it means that large targets will be quickly surfaced in your
@@ -1268,8 +1276,16 @@ def generate_new_examples(self) -> None:
12681276 # the distribution towards small (and less powerful) examples. Flaky
12691277 # and loud health checks are better than silent performance degradation.
12701278 small_example_cap = min (self .settings .max_examples // 10 , 50 )
1271- optimise_at = max (self .settings .max_examples // 2 , small_example_cap + 1 , 10 )
1272- ran_optimisations = False
1279+ # For small budgets we run a single optimisation pass halfway through
1280+ # the budget, which experience shows is roughly optimal. For larger
1281+ # budgets we instead start optimising early and interleave repeated
1282+ # passes with generation - see _should_optimise_now.
1283+ if self .settings .max_examples < 1000 :
1284+ self ._next_optimise_at = max (
1285+ self .settings .max_examples // 2 , small_example_cap + 1 , 10
1286+ )
1287+ else :
1288+ self ._next_optimise_at = max (200 , small_example_cap + 1 )
12731289 self ._switch_to_hypothesis_provider = False
12741290
12751291 while self .should_generate_more ():
@@ -1353,13 +1369,8 @@ def generate_new_examples(self) -> None:
13531369 # actually exhausts our budget: It might finish running and we
13541370 # discover that actually we still could run a bunch more test cases
13551371 # if we want.
1356- if (
1357- self .valid_examples >= max (small_example_cap , optimise_at )
1358- and not ran_optimisations
1359- ):
1360- ran_optimisations = True
1361- self ._current_phase = "target"
1362- self .optimise_targets ()
1372+ if self ._should_optimise_now ():
1373+ self ._run_optimise_pass ()
13631374
13641375 def generate_mutations_from (self , data : ConjectureData | ConjectureResult ) -> None :
13651376 # A thing that is often useful but rarely happens by accident is
@@ -1519,31 +1530,83 @@ def generate_mutations_from(self, data: ConjectureData | ConjectureResult) -> No
15191530 else :
15201531 failed_mutations += 1
15211532
1522- def optimise_targets (self ) -> None :
1533+ def _should_optimise_now (self ) -> bool :
1534+ """Decide whether to run an optimisation pass at this point in generation."""
1535+ if not self .should_optimise :
1536+ return False
1537+ if self .valid_examples >= self ._next_optimise_at :
1538+ return True
1539+ # After the first pass, we should optimize if we have found a new best score from
1540+ # generation and we have the budget to continue optimizing.
1541+ return (
1542+ self ._best_scores_at_last_pass is not None
1543+ and self .settings .max_examples >= 1000
1544+ and self ._target_valid_spent < self .settings .max_examples // 2
1545+ and any (
1546+ score > self ._best_scores_at_last_pass .get (target , NO_SCORE )
1547+ for target , score in self .best_observed_targets .items ()
1548+ )
1549+ )
1550+
1551+ def _run_optimise_pass (self ) -> None :
1552+ """Run one optimisation pass, then schedule the next one."""
1553+ if self .settings .max_examples < 1000 :
1554+ # We'll only ever run a single optimization pass in this case.
1555+ max_valid = None
1556+ else :
1557+ remaining = self .settings .max_examples // 2 - self ._target_valid_spent
1558+ max_valid = self .valid_examples + min (max (200 , remaining // 4 ), remaining )
1559+ self ._current_phase = "target"
1560+ start_valid = self .valid_examples
1561+ improved = False
1562+ try :
1563+ improved = self .optimise_targets (max_valid = max_valid )
1564+ finally :
1565+ spent = self .valid_examples - start_valid
1566+ self ._target_valid_spent += spent
1567+ # Alternate with generation on a fair-share schedule for as long
1568+ # as passes keep finding improvements and budget remains.
1569+ if (
1570+ max_valid is not None
1571+ and improved
1572+ and self ._target_valid_spent < self .settings .max_examples // 2
1573+ ):
1574+ self ._next_optimise_at = self .valid_examples + spent
1575+ else :
1576+ self ._next_optimise_at = math .inf
1577+ self ._best_scores_at_last_pass = dict (self .best_observed_targets )
1578+ self ._current_phase = "generate"
1579+
1580+ def optimise_targets (self , * , max_valid : int | None = None ) -> bool :
15231581 """If any target observations have been made, attempt to optimise them
1524- all."""
1582+ all, stopping early if ``self.valid_examples`` reaches ``max_valid``.
1583+ Returns whether this made any improvements."""
15251584 if not self .should_optimise :
1526- return
1585+ return False
15271586 from hypothesis .internal .conjecture .optimiser import Optimiser
15281587
15291588 # We want to avoid running the optimiser for too long in case we hit
15301589 # an unbounded target score. We start this off fairly conservatively
15311590 # in case interesting examples are easy to find and then ramp it up
15321591 # on an exponential schedule so we don't hamper the optimiser too much
15331592 # if it needs a long time to find good enough improvements.
1593+ improved = False
15341594 max_improvements = 10
15351595 while True :
15361596 prev_calls = self .call_count
15371597
15381598 any_improvements = False
15391599
15401600 for target , data in list (self .best_examples_of_observed_targets .items ()):
1601+ if max_valid is not None and self .valid_examples >= max_valid :
1602+ return improved
15411603 optimiser = Optimiser (
15421604 self , data , target , max_improvements = max_improvements
15431605 )
15441606 optimiser .run ()
15451607 if optimiser .improvements > 0 :
15461608 any_improvements = True
1609+ improved = True
15471610
15481611 if self .interesting_examples :
15491612 break
@@ -1558,6 +1621,7 @@ def optimise_targets(self) -> None:
15581621
15591622 if prev_calls == self .call_count :
15601623 break
1624+ return improved
15611625
15621626 def pareto_optimise (self ) -> None :
15631627 if self .pareto_front is not None :
@@ -1577,7 +1641,7 @@ def _run(self) -> None:
15771641 self ._switch_to_hypothesis_provider = False
15781642 with self ._log_phase_statistics ("generate" ):
15791643 self .generate_new_examples ()
1580- # We normally run the targeting phase mixed in with the generate phase,
1644+ # We normally run the target phase mixed in with the generate phase,
15811645 # but if we've been asked to run it but not generation then we have to
15821646 # run it explicitly on its own here.
15831647 if Phase .generate not in self .settings .phases :
0 commit comments