Skip to content

Commit cfd0a3d

Browse files
committed
Align live fixture accounting with its named population
1 parent a5bc673 commit cfd0a3d

1 file changed

Lines changed: 188 additions & 33 deletions

File tree

‎scripts/test_ci_verify.py‎

Lines changed: 188 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1594,11 +1594,30 @@ def test_native_live_shards_cover_94_rows_and_use_the_profile_test_directory(sel
15941594
self.assertEqual(len(flat), len(set(flat)))
15951595
self.assertIsNone(ci_verify.validate_ctest_shard_union(summaries))
15961596
lost = json.loads(json.dumps(summaries))
1597-
lost[0]['ctestRan'].pop()
1597+
omitted = lost[0]['ctestRan'].pop()
15981598
for key in ('ctestSelected', 'ctestRows', 'ctestTotal'):
15991599
lost[0][key] -= 1
1600-
with self.assertRaisesRegex(ValueError, 'CTest shard union lost or added a row'):
1600+
# Selected-build accounting refuses the unchanged plan first.
1601+
expected_error = (
1602+
'selected build targets/inputs differ from shard execution'
1603+
if profile == 'live-sanitizers' else 'CTest shard union lost or added a row')
1604+
with self.assertRaises(ValueError) as refusal:
16011605
ci_verify.validate_ctest_shard_union(lost)
1606+
self.assertEqual(str(refusal.exception), expected_error)
1607+
if profile == 'live-sanitizers':
1608+
# Independently omit the same row from results and both
1609+
# selections. Keep the producer map, archives, install/build
1610+
# targets, argv and stage proofs: extra build targets are
1611+
# allowed, so only the global union refusal applies.
1612+
lost_from_plan = json.loads(json.dumps(summaries))
1613+
lost_from_plan[0]['ctestRan'].remove(omitted)
1614+
for key in ('ctestSelected', 'ctestRows', 'ctestTotal'):
1615+
lost_from_plan[0][key] -= 1
1616+
for key in ('ctestBuildInventory', 'ctestBuildInventoryAfter'):
1617+
lost_from_plan[0][key]['selectedNames'].remove(omitted)
1618+
with self.assertRaises(ValueError) as refusal:
1619+
ci_verify.validate_ctest_shard_union(lost_from_plan)
1620+
self.assertEqual(str(refusal.exception), 'CTest shard union lost or added a row')
16021621

16031622
def test_native_live_shard_result_accounting_refuses_one_lost_row(self):
16041623
inventory = self.native_live_inventory()
@@ -5714,33 +5733,138 @@ def pinned_probe(self, count: int) -> list:
57145733
self.assertGreaterEqual(len(pinned), count)
57155734
return pinned[:count]
57165735

5736+
def assert_live_population(self, summary, scripted):
5737+
# Expected arithmetic and categories come from the command fixture's
5738+
# input registry, never the Driver's discovery/accounting results.
5739+
registry = set(scripted._live_registry())
5740+
skipped = list(scripted.exits.get('ctest_skipped', ()))
5741+
not_run = list(scripted.exits.get('ctest_not_run', ()))
5742+
disabled = list(scripted.exits.get('ctest_disabled', ()))
5743+
self.assertEqual(scripted.exits['ctest_registered'], len(registry))
5744+
self.assertEqual(scripted.exits['ctest_rows'], len(registry))
5745+
self.assertTrue(set(skipped + not_run + disabled) <= registry)
5746+
executed = registry - set(skipped + not_run + disabled)
5747+
self.assertEqual(summary['ctestRegistered'], len(registry))
5748+
self.assertEqual(summary['ctestSelected'], len(registry))
5749+
self.assertEqual(summary['ctestTotal'], len(registry) - len(disabled))
5750+
self.assertEqual(summary['ctestRows'], len(executed))
5751+
self.assertEqual(summary['ctestSkipped'], skipped)
5752+
self.assertEqual(summary['ctestNotRun'], not_run)
5753+
self.assertEqual(summary['ctestDisabled'], disabled)
5754+
results = summary['ctestBuildResults']
5755+
self.assertEqual(results['selectedNames'], sorted(registry))
5756+
self.assertEqual(results['ran'], sorted(executed))
5757+
self.assertEqual(results['skipped'], sorted(skipped))
5758+
self.assertEqual(results['notRun'], sorted(not_run))
5759+
self.assertEqual(results['disabled'], sorted(disabled))
5760+
return registry, executed
5761+
57175762
def test_green_the_absent_runner_placeholders_skip_and_the_profile_passes(self):
57185763
# Control (a): the public native-live shape. The runner/ tree
57195764
# registers its 65 old pinned placeholders beside the required rows;
5720-
# they skip, ran stays at the floor, and the profile passes with the
5721-
# acceptance printed.
5765+
# they skip and every named executable row runs.
5766+
# expectation corrected: 86 -> 95 registered/total and 21 -> 30
5767+
# executable, because the named registry has 30 executable rows plus
5768+
# 65 pinned skips. The production floor stays 21.
5769+
# Literal names transcribed from Scripted's base row declarations and
5770+
# runner/CMakeLists.txt, including _pf_equivalence_probes and its cover
5771+
# row; independent of discovery, results and runner_e2e_test_names().
5772+
expected_executed = {
5773+
'native_live_chart_calendar_e2e',
5774+
'native_live_chart_input_e2e',
5775+
'native_live_checkpoint_e2e',
5776+
'native_live_e2e',
5777+
'native_live_equivalence_e2e_bracket-trailing-activation-offset-path-01',
5778+
'native_live_equivalence_e2e_order-entry-implicit-reversal-exit-01',
5779+
'native_live_equivalence_e2e_order-harbor-stop-limit-01',
5780+
'native_live_equivalence_e2e_order-keystone-limit-replace-01',
5781+
'native_live_equivalence_e2e_order-stop-entry-touch-boundary-01',
5782+
'native_live_equivalence_e2e_order-tranche-partial-market-close-01',
5783+
'native_live_equivalence_e2e_pyramid-terrace-staged-entry-01',
5784+
'native_live_equivalence_e2e_ta-rsi14-cross-50-01',
5785+
'native_live_genuine_retry_e2e',
5786+
'native_live_help',
5787+
'native_live_ledger_behavior_e2e',
5788+
'native_live_ratio_interval_e2e',
5789+
'native_live_report_e2e',
5790+
'native_live_routing_e2e',
5791+
'native_live_service_e2e',
5792+
'native_live_startup_e2e',
5793+
'native_live_stdin_idle_e2e',
5794+
'native_live_websocket_e2e',
5795+
'test_live_json',
5796+
'test_native_example_batch',
5797+
'test_native_example_selected',
5798+
'test_native_live_equivalence_split',
5799+
'test_native_live_routing',
5800+
'test_native_live_startup',
5801+
'test_native_live_store',
5802+
'test_native_live_websocket',
5803+
}
57225804
pinned = self.pinned_probe(65)
5723-
code, summary, _, build_dir = self.run_profile(
5724-
'live-sanitizers', ctest_registered=65 + LIVE_SANITIZERS_MIN_TESTS,
5725-
ctest_rows=65 + LIVE_SANITIZERS_MIN_TESTS,
5805+
self.assertEqual(len(expected_executed), 30)
5806+
code, summary, scripted, build_dir = self.run_profile(
5807+
'live-sanitizers', ctest_registered=95, ctest_rows=95,
57265808
ctest_skipped=pinned)
5809+
registry, _ = self.assert_live_population(summary, scripted)
5810+
self.assertEqual(registry - set(pinned), expected_executed)
5811+
expected_count = len(registry - set(pinned))
57275812
self.assertEqual(code, 0, summary['failures'])
5728-
self.assertEqual(summary['ctestRows'], LIVE_SANITIZERS_MIN_TESTS)
5813+
self.assertEqual(failure_stages(summary), [])
5814+
self.assertEqual(summary['ctestRegistered'], 95)
5815+
self.assertEqual(summary['ctestTotal'], 95)
5816+
self.assertEqual(summary['ctestRows'], expected_count)
5817+
self.assertEqual(LIVE_SANITIZERS_MIN_TESTS, 21)
5818+
self.assertEqual(summary['minTests'], 21)
5819+
self.assertGreaterEqual(summary['ctestRows'], 21)
5820+
self.assertEqual(len(summary['ctestSkipped']), 65)
5821+
self.assertEqual(summary['ctestSkipped'], pinned)
5822+
results = summary['ctestBuildResults']
5823+
self.assertEqual(results['ran'], sorted(expected_executed))
5824+
self.assertEqual(results['skipped'], pinned)
57295825
self.assertNotIn('live-test-skips', failure_stages(summary))
57305826
self.assertIn('live-test-skips', stage_names(summary))
57315827
log = (build_dir / 'ci-logs' / 'live-test-skips.log').read_text()
57325828
self.assertIn('65 pinned private-fixture rows skipping', log)
57335829

5830+
# The native four-row override control is a different population.
5831+
# Pin both sides of the boundary on this same 95-name registry,
5832+
# retaining every executable name and all 65 pinned skips.
5833+
for minimum, expected_code, stages in (
5834+
(expected_count, 0, []),
5835+
(expected_count + 1, 1, ['ctest-floor'])):
5836+
with self.subTest(min_tests=minimum):
5837+
code, summary, scripted, _ = self.run_profile(
5838+
'live-sanitizers', extra=['--min-tests', str(minimum)],
5839+
ctest_registered=len(registry), ctest_rows=len(registry),
5840+
ctest_skipped=pinned)
5841+
boundary_registry, boundary_executed = self.assert_live_population(
5842+
summary, scripted)
5843+
self.assertEqual(boundary_registry, registry)
5844+
self.assertEqual(boundary_executed, expected_executed)
5845+
self.assertEqual(summary['ctestBuildResults']['ran'], sorted(expected_executed))
5846+
self.assertEqual(summary['ctestSkipped'], pinned)
5847+
self.assertEqual(summary['minTests'], minimum)
5848+
self.assertEqual(code, expected_code, summary['failures'])
5849+
self.assertEqual(failure_stages(summary), stages)
5850+
if expected_code:
5851+
self.assertIn(
5852+
f'ctest ran {expected_count} rows, below the floor of {minimum} '
5853+
'for the live-sanitizers profile',
5854+
summary['failures'][0]['error'])
5855+
57345856
def test_red_an_unpinned_runner_skip_fails(self):
57355857
# Control (b): one skipped row outside the pin is a failure, even
57365858
# though the arithmetic alone would tolerate it.
57375859
pinned = self.pinned_probe(65)
5738-
code, summary, _, _ = self.run_profile(
5739-
'live-sanitizers', ctest_registered=66 + LIVE_SANITIZERS_MIN_TESTS,
5740-
ctest_rows=66 + LIVE_SANITIZERS_MIN_TESTS,
5860+
# expectation corrected: 87 -> 96, because 30 executable names plus
5861+
# 65 pinned skips and one unpinned skip must all be represented.
5862+
code, summary, scripted, _ = self.run_profile(
5863+
'live-sanitizers', ctest_registered=96, ctest_rows=96,
57415864
ctest_skipped=pinned + ['test_unpinned_runner_row'])
5865+
self.assert_live_population(summary, scripted)
57425866
self.assertEqual(code, 1)
5743-
self.assertIn('live-test-skips', failure_stages(summary))
5867+
self.assertEqual(failure_stages(summary), ['live-test-skips'])
57445868
error = next(item['error'] for item in summary['failures']
57455869
if item['stage'] == 'live-test-skips')
57465870
self.assertIn('test_unpinned_runner_row', error)
@@ -5750,39 +5874,58 @@ def test_red_a_not_run_row_fails_with_the_pinned_skips(self):
57505874
# Control (c): a row CTest could not start is refused beside the
57515875
# pinned skips -- the acceptance covers skips, nothing else.
57525876
pinned = self.pinned_probe(65)
5753-
code, summary, _, _ = self.run_profile(
5754-
'live-sanitizers', ctest_registered=66 + LIVE_SANITIZERS_MIN_TESTS,
5755-
ctest_rows=66 + LIVE_SANITIZERS_MIN_TESTS,
5877+
# expectation corrected: 87 -> 96, because the named registry also
5878+
# carries the extra not-run row beside 30 executable names and 65 skips.
5879+
code, summary, scripted, _ = self.run_profile(
5880+
'live-sanitizers', ctest_registered=96, ctest_rows=96,
57565881
ctest_skipped=pinned, ctest_not_run=['test_live_not_run_row'])
5882+
self.assert_live_population(summary, scripted)
57575883
self.assertEqual(code, 1)
5758-
self.assertIn('live-test-skips', failure_stages(summary))
5884+
self.assertEqual(failure_stages(summary),
5885+
['ctest-build-accounting', 'live-test-skips'])
5886+
self.assertEqual(summary['failures'][0]['error'],
5887+
'selected build contains not-run or disabled rows')
5888+
self.assertEqual(summary['failures'][1]['error'],
5889+
'live-sanitizers cannot accept skipped, disabled or unreadable rows')
57595890
error = next(item['error'] for item in summary['failures']
57605891
if item['stage'] == 'live-test-skips')
57615892
self.assertIn('cannot accept skipped, disabled or unreadable rows', error)
57625893

57635894
def test_red_a_disabled_row_fails_with_the_pinned_skips(self):
57645895
# Control (c) again, for the disabled list: CTest leaves it out of its
5765-
# own count, so only this rule can refuse it.
5896+
# own count. Named build accounting and the live skip rule both
5897+
# deliberately refuse the disabled result, with no population mismatch.
57665898
pinned = self.pinned_probe(65)
5767-
code, summary, _, _ = self.run_profile(
5768-
'live-sanitizers', ctest_registered=65 + LIVE_SANITIZERS_MIN_TESTS,
5769-
ctest_rows=65 + LIVE_SANITIZERS_MIN_TESTS,
5899+
# expectation corrected: 86 -> 96 registered/input rows, because the
5900+
# disabled name also belongs to the registry; CTest's total is 95.
5901+
code, summary, scripted, _ = self.run_profile(
5902+
'live-sanitizers', ctest_registered=96, ctest_rows=96,
57705903
ctest_skipped=pinned, ctest_disabled=['test_live_disabled_row'])
5904+
self.assert_live_population(summary, scripted)
57715905
self.assertEqual(code, 1)
5772-
self.assertIn('live-test-skips', failure_stages(summary))
5906+
self.assertEqual(failure_stages(summary),
5907+
['ctest-build-accounting', 'live-test-skips'])
5908+
self.assertEqual(summary['failures'][0]['error'],
5909+
'selected build contains not-run or disabled rows')
5910+
self.assertEqual(summary['failures'][1]['error'],
5911+
'live-sanitizers cannot accept skipped, disabled or unreadable rows')
57735912

57745913
def test_red_overlay_present_a_pinned_skip_still_fails(self):
57755914
# Control (d): with the overlay present the old rule stands -- every
57765915
# runner row must run -- so a pinned row that skips fails on the
57775916
# private-fixtures rule and on this one.
5917+
pinned = self.pinned_probe(1)
5918+
# expectation corrected: 22 -> 31, because the fixture declares
5919+
# 30 executable names plus the one pinned skip in this control.
57785920
with mock.patch.dict(os.environ, {'PINEFORGE_PRIVATE_FIXTURES': '1'}):
5779-
code, summary, _, _ = self.run_profile(
5780-
'live-sanitizers', ctest_registered=LIVE_SANITIZERS_MIN_TESTS + 1,
5781-
ctest_rows=LIVE_SANITIZERS_MIN_TESTS + 1,
5782-
ctest_skipped=self.pinned_probe(1))
5921+
code, summary, scripted, _ = self.run_profile(
5922+
'live-sanitizers', ctest_registered=31, ctest_rows=31,
5923+
ctest_skipped=pinned)
5924+
self.assert_live_population(summary, scripted)
57835925
self.assertEqual(code, 1)
5784-
self.assertIn('live-test-skips', failure_stages(summary))
5785-
self.assertIn('ctest-private-fixtures', failure_stages(summary))
5926+
self.assertEqual(failure_stages(summary),
5927+
['ctest-private-fixtures', 'live-test-skips'])
5928+
self.assertIn(pinned[0], summary['failures'][0]['error'])
57865929
error = next(item['error'] for item in summary['failures']
57875930
if item['stage'] == 'live-test-skips')
57885931
self.assertIn('cannot accept skipped', error)
@@ -5791,11 +5934,16 @@ def test_green_live_tsan_carries_the_same_rule(self):
57915934
# The rule keys on the profile SET: live-tsan accepts the pinned
57925935
# placeholders the same way.
57935936
pinned = self.pinned_probe(65)
5794-
code, summary, _, _ = self.run_profile(
5937+
code, summary, scripted, _ = self.run_profile(
57955938
'live-tsan', ctest_registered=65 + LIVE_SANITIZERS_MIN_TESTS,
57965939
ctest_rows=65 + LIVE_SANITIZERS_MIN_TESTS,
57975940
ctest_skipped=pinned)
57985941
self.assertEqual(code, 0, summary['failures'])
5942+
self.assertEqual(failure_stages(summary), [])
5943+
self.assertEqual(summary['ctestRegistered'], scripted.exits['ctest_registered'])
5944+
self.assertEqual(summary['ctestTotal'], scripted.exits['ctest_rows'])
5945+
self.assertEqual(summary['ctestRows'], scripted.exits['ctest_rows'] - len(pinned))
5946+
self.assertEqual(summary['ctestSkipped'], pinned)
57995947
self.assertIn('live-test-skips', stage_names(summary))
58005948

58015949
def test_required_equivalence_skip_fails_even_when_other_rows_cover_the_floor(self):
@@ -5806,13 +5954,20 @@ def test_required_equivalence_skip_fails_even_when_other_rows_cover_the_floor(se
58065954
for probe in probes:
58075955
name = 'native_live_equivalence_e2e_' + probe
58085956
with self.subTest(profile=profile, probe=probe):
5809-
code, summary, _, _ = self.run_profile(
5810-
profile, ctest_registered=LIVE_SANITIZERS_MIN_TESTS + 1,
5811-
ctest_rows=LIVE_SANITIZERS_MIN_TESTS + 1,
5957+
# expectation corrected: 22 -> 30 for live-sanitizers,
5958+
# because the skipped required name is already in its
5959+
# 30-name registry. The live-tsan count fixture is unchanged.
5960+
registered = (30 if profile == 'live-sanitizers'
5961+
else LIVE_SANITIZERS_MIN_TESTS + 1)
5962+
code, summary, scripted, _ = self.run_profile(
5963+
profile, ctest_registered=registered, ctest_rows=registered,
58125964
ctest_skipped=[name])
5965+
if profile == 'live-sanitizers':
5966+
self.assert_live_population(summary, scripted)
5967+
else:
5968+
self.assertEqual(summary['ctestRows'], LIVE_SANITIZERS_MIN_TESTS)
58135969
self.assertEqual(code, 1)
5814-
self.assertEqual(summary['ctestRows'], LIVE_SANITIZERS_MIN_TESTS)
5815-
self.assertIn('live-test-skips', failure_stages(summary))
5970+
self.assertEqual(failure_stages(summary), ['live-test-skips'])
58165971
error = next(item['error'] for item in summary['failures']
58175972
if item['stage'] == 'live-test-skips')
58185973
self.assertIn('required runner e2e rows skipped', error)

0 commit comments

Comments
 (0)