@@ -1594,11 +1594,30 @@ def test_native_live_shards_cover_94_rows_and_use_the_profile_test_directory(sel
15941594 self .assertEqual (len (flat ), len (set (flat )))
15951595 self .assertIsNone (ci_verify .validate_ctest_shard_union (summaries ))
15961596 lost = json .loads (json .dumps (summaries ))
1597- lost [0 ]['ctestRan' ].pop ()
1597+ omitted = lost [0 ]['ctestRan' ].pop ()
15981598 for key in ('ctestSelected' , 'ctestRows' , 'ctestTotal' ):
15991599 lost [0 ][key ] -= 1
1600- with self .assertRaisesRegex (ValueError , 'CTest shard union lost or added a row' ):
1600+ # Selected-build accounting refuses the unchanged plan first.
1601+ expected_error = (
1602+ 'selected build targets/inputs differ from shard execution'
1603+ if profile == 'live-sanitizers' else 'CTest shard union lost or added a row' )
1604+ with self .assertRaises (ValueError ) as refusal :
16011605 ci_verify .validate_ctest_shard_union (lost )
1606+ self .assertEqual (str (refusal .exception ), expected_error )
1607+ if profile == 'live-sanitizers' :
1608+ # Independently omit the same row from results and both
1609+ # selections. Keep the producer map, archives, install/build
1610+ # targets, argv and stage proofs: extra build targets are
1611+ # allowed, so only the global union refusal applies.
1612+ lost_from_plan = json .loads (json .dumps (summaries ))
1613+ lost_from_plan [0 ]['ctestRan' ].remove (omitted )
1614+ for key in ('ctestSelected' , 'ctestRows' , 'ctestTotal' ):
1615+ lost_from_plan [0 ][key ] -= 1
1616+ for key in ('ctestBuildInventory' , 'ctestBuildInventoryAfter' ):
1617+ lost_from_plan [0 ][key ]['selectedNames' ].remove (omitted )
1618+ with self .assertRaises (ValueError ) as refusal :
1619+ ci_verify .validate_ctest_shard_union (lost_from_plan )
1620+ self .assertEqual (str (refusal .exception ), 'CTest shard union lost or added a row' )
16021621
16031622 def test_native_live_shard_result_accounting_refuses_one_lost_row (self ):
16041623 inventory = self .native_live_inventory ()
@@ -5714,33 +5733,138 @@ def pinned_probe(self, count: int) -> list:
57145733 self .assertGreaterEqual (len (pinned ), count )
57155734 return pinned [:count ]
57165735
5736+ def assert_live_population (self , summary , scripted ):
5737+ # Expected arithmetic and categories come from the command fixture's
5738+ # input registry, never the Driver's discovery/accounting results.
5739+ registry = set (scripted ._live_registry ())
5740+ skipped = list (scripted .exits .get ('ctest_skipped' , ()))
5741+ not_run = list (scripted .exits .get ('ctest_not_run' , ()))
5742+ disabled = list (scripted .exits .get ('ctest_disabled' , ()))
5743+ self .assertEqual (scripted .exits ['ctest_registered' ], len (registry ))
5744+ self .assertEqual (scripted .exits ['ctest_rows' ], len (registry ))
5745+ self .assertTrue (set (skipped + not_run + disabled ) <= registry )
5746+ executed = registry - set (skipped + not_run + disabled )
5747+ self .assertEqual (summary ['ctestRegistered' ], len (registry ))
5748+ self .assertEqual (summary ['ctestSelected' ], len (registry ))
5749+ self .assertEqual (summary ['ctestTotal' ], len (registry ) - len (disabled ))
5750+ self .assertEqual (summary ['ctestRows' ], len (executed ))
5751+ self .assertEqual (summary ['ctestSkipped' ], skipped )
5752+ self .assertEqual (summary ['ctestNotRun' ], not_run )
5753+ self .assertEqual (summary ['ctestDisabled' ], disabled )
5754+ results = summary ['ctestBuildResults' ]
5755+ self .assertEqual (results ['selectedNames' ], sorted (registry ))
5756+ self .assertEqual (results ['ran' ], sorted (executed ))
5757+ self .assertEqual (results ['skipped' ], sorted (skipped ))
5758+ self .assertEqual (results ['notRun' ], sorted (not_run ))
5759+ self .assertEqual (results ['disabled' ], sorted (disabled ))
5760+ return registry , executed
5761+
57175762 def test_green_the_absent_runner_placeholders_skip_and_the_profile_passes (self ):
57185763 # Control (a): the public native-live shape. The runner/ tree
57195764 # registers its 65 old pinned placeholders beside the required rows;
5720- # they skip, ran stays at the floor, and the profile passes with the
5721- # acceptance printed.
5765+ # they skip and every named executable row runs.
5766+ # expectation corrected: 86 -> 95 registered/total and 21 -> 30
5767+ # executable, because the named registry has 30 executable rows plus
5768+ # 65 pinned skips. The production floor stays 21.
5769+ # Literal names transcribed from Scripted's base row declarations and
5770+ # runner/CMakeLists.txt, including _pf_equivalence_probes and its cover
5771+ # row; independent of discovery, results and runner_e2e_test_names().
5772+ expected_executed = {
5773+ 'native_live_chart_calendar_e2e' ,
5774+ 'native_live_chart_input_e2e' ,
5775+ 'native_live_checkpoint_e2e' ,
5776+ 'native_live_e2e' ,
5777+ 'native_live_equivalence_e2e_bracket-trailing-activation-offset-path-01' ,
5778+ 'native_live_equivalence_e2e_order-entry-implicit-reversal-exit-01' ,
5779+ 'native_live_equivalence_e2e_order-harbor-stop-limit-01' ,
5780+ 'native_live_equivalence_e2e_order-keystone-limit-replace-01' ,
5781+ 'native_live_equivalence_e2e_order-stop-entry-touch-boundary-01' ,
5782+ 'native_live_equivalence_e2e_order-tranche-partial-market-close-01' ,
5783+ 'native_live_equivalence_e2e_pyramid-terrace-staged-entry-01' ,
5784+ 'native_live_equivalence_e2e_ta-rsi14-cross-50-01' ,
5785+ 'native_live_genuine_retry_e2e' ,
5786+ 'native_live_help' ,
5787+ 'native_live_ledger_behavior_e2e' ,
5788+ 'native_live_ratio_interval_e2e' ,
5789+ 'native_live_report_e2e' ,
5790+ 'native_live_routing_e2e' ,
5791+ 'native_live_service_e2e' ,
5792+ 'native_live_startup_e2e' ,
5793+ 'native_live_stdin_idle_e2e' ,
5794+ 'native_live_websocket_e2e' ,
5795+ 'test_live_json' ,
5796+ 'test_native_example_batch' ,
5797+ 'test_native_example_selected' ,
5798+ 'test_native_live_equivalence_split' ,
5799+ 'test_native_live_routing' ,
5800+ 'test_native_live_startup' ,
5801+ 'test_native_live_store' ,
5802+ 'test_native_live_websocket' ,
5803+ }
57225804 pinned = self .pinned_probe (65 )
5723- code , summary , _ , build_dir = self .run_profile (
5724- 'live-sanitizers' , ctest_registered = 65 + LIVE_SANITIZERS_MIN_TESTS ,
5725- ctest_rows = 65 + LIVE_SANITIZERS_MIN_TESTS ,
5805+ self .assertEqual ( len ( expected_executed ), 30 )
5806+ code , summary , scripted , build_dir = self . run_profile (
5807+ 'live-sanitizers' , ctest_registered = 95 , ctest_rows = 95 ,
57265808 ctest_skipped = pinned )
5809+ registry , _ = self .assert_live_population (summary , scripted )
5810+ self .assertEqual (registry - set (pinned ), expected_executed )
5811+ expected_count = len (registry - set (pinned ))
57275812 self .assertEqual (code , 0 , summary ['failures' ])
5728- self .assertEqual (summary ['ctestRows' ], LIVE_SANITIZERS_MIN_TESTS )
5813+ self .assertEqual (failure_stages (summary ), [])
5814+ self .assertEqual (summary ['ctestRegistered' ], 95 )
5815+ self .assertEqual (summary ['ctestTotal' ], 95 )
5816+ self .assertEqual (summary ['ctestRows' ], expected_count )
5817+ self .assertEqual (LIVE_SANITIZERS_MIN_TESTS , 21 )
5818+ self .assertEqual (summary ['minTests' ], 21 )
5819+ self .assertGreaterEqual (summary ['ctestRows' ], 21 )
5820+ self .assertEqual (len (summary ['ctestSkipped' ]), 65 )
5821+ self .assertEqual (summary ['ctestSkipped' ], pinned )
5822+ results = summary ['ctestBuildResults' ]
5823+ self .assertEqual (results ['ran' ], sorted (expected_executed ))
5824+ self .assertEqual (results ['skipped' ], pinned )
57295825 self .assertNotIn ('live-test-skips' , failure_stages (summary ))
57305826 self .assertIn ('live-test-skips' , stage_names (summary ))
57315827 log = (build_dir / 'ci-logs' / 'live-test-skips.log' ).read_text ()
57325828 self .assertIn ('65 pinned private-fixture rows skipping' , log )
57335829
5830+ # The native four-row override control is a different population.
5831+ # Pin both sides of the boundary on this same 95-name registry,
5832+ # retaining every executable name and all 65 pinned skips.
5833+ for minimum , expected_code , stages in (
5834+ (expected_count , 0 , []),
5835+ (expected_count + 1 , 1 , ['ctest-floor' ])):
5836+ with self .subTest (min_tests = minimum ):
5837+ code , summary , scripted , _ = self .run_profile (
5838+ 'live-sanitizers' , extra = ['--min-tests' , str (minimum )],
5839+ ctest_registered = len (registry ), ctest_rows = len (registry ),
5840+ ctest_skipped = pinned )
5841+ boundary_registry , boundary_executed = self .assert_live_population (
5842+ summary , scripted )
5843+ self .assertEqual (boundary_registry , registry )
5844+ self .assertEqual (boundary_executed , expected_executed )
5845+ self .assertEqual (summary ['ctestBuildResults' ]['ran' ], sorted (expected_executed ))
5846+ self .assertEqual (summary ['ctestSkipped' ], pinned )
5847+ self .assertEqual (summary ['minTests' ], minimum )
5848+ self .assertEqual (code , expected_code , summary ['failures' ])
5849+ self .assertEqual (failure_stages (summary ), stages )
5850+ if expected_code :
5851+ self .assertIn (
5852+ f'ctest ran { expected_count } rows, below the floor of { minimum } '
5853+ 'for the live-sanitizers profile' ,
5854+ summary ['failures' ][0 ]['error' ])
5855+
57345856 def test_red_an_unpinned_runner_skip_fails (self ):
57355857 # Control (b): one skipped row outside the pin is a failure, even
57365858 # though the arithmetic alone would tolerate it.
57375859 pinned = self .pinned_probe (65 )
5738- code , summary , _ , _ = self .run_profile (
5739- 'live-sanitizers' , ctest_registered = 66 + LIVE_SANITIZERS_MIN_TESTS ,
5740- ctest_rows = 66 + LIVE_SANITIZERS_MIN_TESTS ,
5860+ # expectation corrected: 87 -> 96, because 30 executable names plus
5861+ # 65 pinned skips and one unpinned skip must all be represented.
5862+ code , summary , scripted , _ = self .run_profile (
5863+ 'live-sanitizers' , ctest_registered = 96 , ctest_rows = 96 ,
57415864 ctest_skipped = pinned + ['test_unpinned_runner_row' ])
5865+ self .assert_live_population (summary , scripted )
57425866 self .assertEqual (code , 1 )
5743- self .assertIn ( 'live-test-skips' , failure_stages ( summary ) )
5867+ self .assertEqual ( failure_stages ( summary ), [ 'live-test-skips' ] )
57445868 error = next (item ['error' ] for item in summary ['failures' ]
57455869 if item ['stage' ] == 'live-test-skips' )
57465870 self .assertIn ('test_unpinned_runner_row' , error )
@@ -5750,39 +5874,58 @@ def test_red_a_not_run_row_fails_with_the_pinned_skips(self):
57505874 # Control (c): a row CTest could not start is refused beside the
57515875 # pinned skips -- the acceptance covers skips, nothing else.
57525876 pinned = self .pinned_probe (65 )
5753- code , summary , _ , _ = self .run_profile (
5754- 'live-sanitizers' , ctest_registered = 66 + LIVE_SANITIZERS_MIN_TESTS ,
5755- ctest_rows = 66 + LIVE_SANITIZERS_MIN_TESTS ,
5877+ # expectation corrected: 87 -> 96, because the named registry also
5878+ # carries the extra not-run row beside 30 executable names and 65 skips.
5879+ code , summary , scripted , _ = self .run_profile (
5880+ 'live-sanitizers' , ctest_registered = 96 , ctest_rows = 96 ,
57565881 ctest_skipped = pinned , ctest_not_run = ['test_live_not_run_row' ])
5882+ self .assert_live_population (summary , scripted )
57575883 self .assertEqual (code , 1 )
5758- self .assertIn ('live-test-skips' , failure_stages (summary ))
5884+ self .assertEqual (failure_stages (summary ),
5885+ ['ctest-build-accounting' , 'live-test-skips' ])
5886+ self .assertEqual (summary ['failures' ][0 ]['error' ],
5887+ 'selected build contains not-run or disabled rows' )
5888+ self .assertEqual (summary ['failures' ][1 ]['error' ],
5889+ 'live-sanitizers cannot accept skipped, disabled or unreadable rows' )
57595890 error = next (item ['error' ] for item in summary ['failures' ]
57605891 if item ['stage' ] == 'live-test-skips' )
57615892 self .assertIn ('cannot accept skipped, disabled or unreadable rows' , error )
57625893
57635894 def test_red_a_disabled_row_fails_with_the_pinned_skips (self ):
57645895 # Control (c) again, for the disabled list: CTest leaves it out of its
5765- # own count, so only this rule can refuse it.
5896+ # own count. Named build accounting and the live skip rule both
5897+ # deliberately refuse the disabled result, with no population mismatch.
57665898 pinned = self .pinned_probe (65 )
5767- code , summary , _ , _ = self .run_profile (
5768- 'live-sanitizers' , ctest_registered = 65 + LIVE_SANITIZERS_MIN_TESTS ,
5769- ctest_rows = 65 + LIVE_SANITIZERS_MIN_TESTS ,
5899+ # expectation corrected: 86 -> 96 registered/input rows, because the
5900+ # disabled name also belongs to the registry; CTest's total is 95.
5901+ code , summary , scripted , _ = self .run_profile (
5902+ 'live-sanitizers' , ctest_registered = 96 , ctest_rows = 96 ,
57705903 ctest_skipped = pinned , ctest_disabled = ['test_live_disabled_row' ])
5904+ self .assert_live_population (summary , scripted )
57715905 self .assertEqual (code , 1 )
5772- self .assertIn ('live-test-skips' , failure_stages (summary ))
5906+ self .assertEqual (failure_stages (summary ),
5907+ ['ctest-build-accounting' , 'live-test-skips' ])
5908+ self .assertEqual (summary ['failures' ][0 ]['error' ],
5909+ 'selected build contains not-run or disabled rows' )
5910+ self .assertEqual (summary ['failures' ][1 ]['error' ],
5911+ 'live-sanitizers cannot accept skipped, disabled or unreadable rows' )
57735912
57745913 def test_red_overlay_present_a_pinned_skip_still_fails (self ):
57755914 # Control (d): with the overlay present the old rule stands -- every
57765915 # runner row must run -- so a pinned row that skips fails on the
57775916 # private-fixtures rule and on this one.
5917+ pinned = self .pinned_probe (1 )
5918+ # expectation corrected: 22 -> 31, because the fixture declares
5919+ # 30 executable names plus the one pinned skip in this control.
57785920 with mock .patch .dict (os .environ , {'PINEFORGE_PRIVATE_FIXTURES' : '1' }):
5779- code , summary , _ , _ = self .run_profile (
5780- 'live-sanitizers' , ctest_registered = LIVE_SANITIZERS_MIN_TESTS + 1 ,
5781- ctest_rows = LIVE_SANITIZERS_MIN_TESTS + 1 ,
5782- ctest_skipped = self .pinned_probe ( 1 ) )
5921+ code , summary , scripted , _ = self .run_profile (
5922+ 'live-sanitizers' , ctest_registered = 31 , ctest_rows = 31 ,
5923+ ctest_skipped = pinned )
5924+ self .assert_live_population ( summary , scripted )
57835925 self .assertEqual (code , 1 )
5784- self .assertIn ('live-test-skips' , failure_stages (summary ))
5785- self .assertIn ('ctest-private-fixtures' , failure_stages (summary ))
5926+ self .assertEqual (failure_stages (summary ),
5927+ ['ctest-private-fixtures' , 'live-test-skips' ])
5928+ self .assertIn (pinned [0 ], summary ['failures' ][0 ]['error' ])
57865929 error = next (item ['error' ] for item in summary ['failures' ]
57875930 if item ['stage' ] == 'live-test-skips' )
57885931 self .assertIn ('cannot accept skipped' , error )
@@ -5791,11 +5934,16 @@ def test_green_live_tsan_carries_the_same_rule(self):
57915934 # The rule keys on the profile SET: live-tsan accepts the pinned
57925935 # placeholders the same way.
57935936 pinned = self .pinned_probe (65 )
5794- code , summary , _ , _ = self .run_profile (
5937+ code , summary , scripted , _ = self .run_profile (
57955938 'live-tsan' , ctest_registered = 65 + LIVE_SANITIZERS_MIN_TESTS ,
57965939 ctest_rows = 65 + LIVE_SANITIZERS_MIN_TESTS ,
57975940 ctest_skipped = pinned )
57985941 self .assertEqual (code , 0 , summary ['failures' ])
5942+ self .assertEqual (failure_stages (summary ), [])
5943+ self .assertEqual (summary ['ctestRegistered' ], scripted .exits ['ctest_registered' ])
5944+ self .assertEqual (summary ['ctestTotal' ], scripted .exits ['ctest_rows' ])
5945+ self .assertEqual (summary ['ctestRows' ], scripted .exits ['ctest_rows' ] - len (pinned ))
5946+ self .assertEqual (summary ['ctestSkipped' ], pinned )
57995947 self .assertIn ('live-test-skips' , stage_names (summary ))
58005948
58015949 def test_required_equivalence_skip_fails_even_when_other_rows_cover_the_floor (self ):
@@ -5806,13 +5954,20 @@ def test_required_equivalence_skip_fails_even_when_other_rows_cover_the_floor(se
58065954 for probe in probes :
58075955 name = 'native_live_equivalence_e2e_' + probe
58085956 with self .subTest (profile = profile , probe = probe ):
5809- code , summary , _ , _ = self .run_profile (
5810- profile , ctest_registered = LIVE_SANITIZERS_MIN_TESTS + 1 ,
5811- ctest_rows = LIVE_SANITIZERS_MIN_TESTS + 1 ,
5957+ # expectation corrected: 22 -> 30 for live-sanitizers,
5958+ # because the skipped required name is already in its
5959+ # 30-name registry. The live-tsan count fixture is unchanged.
5960+ registered = (30 if profile == 'live-sanitizers'
5961+ else LIVE_SANITIZERS_MIN_TESTS + 1 )
5962+ code , summary , scripted , _ = self .run_profile (
5963+ profile , ctest_registered = registered , ctest_rows = registered ,
58125964 ctest_skipped = [name ])
5965+ if profile == 'live-sanitizers' :
5966+ self .assert_live_population (summary , scripted )
5967+ else :
5968+ self .assertEqual (summary ['ctestRows' ], LIVE_SANITIZERS_MIN_TESTS )
58135969 self .assertEqual (code , 1 )
5814- self .assertEqual (summary ['ctestRows' ], LIVE_SANITIZERS_MIN_TESTS )
5815- self .assertIn ('live-test-skips' , failure_stages (summary ))
5970+ self .assertEqual (failure_stages (summary ), ['live-test-skips' ])
58165971 error = next (item ['error' ] for item in summary ['failures' ]
58175972 if item ['stage' ] == 'live-test-skips' )
58185973 self .assertIn ('required runner e2e rows skipped' , error )
0 commit comments