2222
2323The fixture downloads only JUnit. --junit-classpath accepts local JUnit jars for
2424offline checks. --work-dir retains the fixture, logs, and comparison results.
25+ --github-output exports the workflow matrix/count from one shared definition.
26+ --manifests verifies the real inventories downloaded from an Iceberg CI run.
2527"""
2628
2729import argparse
3638
3739
3840INIT_SCRIPT = Path (__file__ ).with_name ("iceberg-test-shards.gradle" )
41+ # The workflow matrix and Gradle's partition count must come from the same value,
42+ # not strategy.job-total (which includes any other matrix dimensions).
43+ SHARD_COUNT = 4
44+
45+
46+ def verify_manifests (root , task ):
47+ """Check each latest shard attempt against its independently captured baseline.
48+
49+ Rerunning only failed jobs leaves successful jobs' artifacts in an earlier
50+ attempt. Keep those, but never count two attempts of the same shard twice.
51+ Artifact download is scoped to one run and Iceberg/Spark/Scala/JDK tuple.
52+ """
53+ attempts = {}
54+ for path in sorted (root .rglob ("*.json" )):
55+ manifest = json .loads (path .read_text ())
56+ for key in ("shard" , "count" , "attempt" ):
57+ if type (manifest .get (key )) is not int or manifest [key ] < 1 :
58+ raise ValueError (f"{ path } : { key } must be a positive integer" )
59+ key = manifest ["shard" ], manifest ["attempt" ]
60+ if key in attempts :
61+ raise ValueError (f"Duplicate inventory for shard/attempt { key } : { path } " )
62+ attempts [key ] = manifest
63+
64+ expected = set (range (1 , SHARD_COUNT + 1 ))
65+ indices = {index for index , _ in attempts }
66+ if indices != expected :
67+ raise ValueError (f"Expected shard indices { sorted (expected )} , found { sorted (indices )} " )
68+
69+ baseline = None
70+ combined = Counter ()
71+ for index in sorted (expected ):
72+ attempt = max (attempt for shard , attempt in attempts if shard == index )
73+ manifest = attempts [index , attempt ]
74+ if manifest .get ("task" ) != task or manifest ["count" ] != SHARD_COUNT :
75+ raise ValueError (f"Shard { index } : mismatched task or shard count" )
76+ for field in ("candidates" , "unshardedCandidates" ):
77+ values = manifest .get (field )
78+ if not isinstance (values , list ) or not all (isinstance (v , str ) for v in values ):
79+ raise ValueError (f"Shard { index } : { field } must be a list of class paths" )
80+ if len (set (values )) != len (values ):
81+ raise ValueError (f"Shard { index } : duplicate class paths in { field } " )
82+ inventory = set (manifest ["unshardedCandidates" ])
83+ if not inventory :
84+ raise ValueError (f"Shard { index } : empty unsharded candidate inventory" )
85+ if baseline is not None and baseline != inventory :
86+ raise ValueError (f"Shard { index } : unsharded inventories disagree" )
87+ baseline = inventory
88+ combined .update (manifest ["candidates" ])
89+
90+ if set (combined ) != baseline :
91+ missing , extra = baseline - set (combined ), set (combined ) - baseline
92+ raise ValueError (f"Shard coverage mismatch: missing={ sorted (missing )} , extra={ sorted (extra )} " )
93+ duplicates = sorted (name for name , count in combined .items () if count != 1 )
94+ if duplicates :
95+ raise ValueError (f"Candidate classes selected by multiple shards: { duplicates } " )
96+ print (f"Verified { SHARD_COUNT } shards and { len (baseline )} candidate classes: "
97+ "the selected inventories equal the unsharded inventory exactly once." , flush = True )
98+
99+
100+ def write_workflow_matrix (output ):
101+ with output .open ("a" ) as stream :
102+ stream .write (f"matrix={ json .dumps ({'shard' : list (range (1 , SHARD_COUNT + 1 ))})} \n " )
103+ stream .write (f"count={ SHARD_COUNT } \n " )
39104
40105
41106def groovy_string (value ):
@@ -168,7 +233,7 @@ def run(label, extra=(), task="test", expect_failure=False):
168233 print (f"{ label } : { 'expected failure' if expect_failure else 'passed' } " , flush = True )
169234 return read_cases (root , task )
170235
171- def shard_args (index , count = 4 , task = ":test" ):
236+ def shard_args (index , count = SHARD_COUNT , task = ":test" ):
172237 return ["--init-script" , str (INIT_SCRIPT .resolve ()), f"-PcometShardTask={ task } " ,
173238 f"-PcometShardIndex={ index } " , f"-PcometShardCount={ count } " ]
174239
@@ -183,11 +248,12 @@ def shard_args(index, count=4, task=":test"):
183248 combined_cases = Counter ()
184249 combined_candidates = Counter ()
185250 failure_owner = None
186- for index in range (1 , 5 ):
251+ for index in range (1 , SHARD_COUNT + 1 ):
187252 cases = run (f"shard-{ index } " , shard_args (index ))
188253 manifest = json .loads ((root / f"build/comet-shards/test-{ index } .json" ).read_text ())
189254 candidates = json .loads ((root / "build/candidates.json" ).read_text ())
190255 assert candidates == manifest ["candidates" ]
256+ assert set (manifest ["unshardedCandidates" ]) == inventory
191257 assert cases , f"empty fixture shard { index } "
192258 combined_cases .update (cases )
193259 combined_candidates .update (candidates )
@@ -197,6 +263,7 @@ def shard_args(index, count=4, task=":test"):
197263 assert set (combined_candidates ) == inventory , (set (combined_candidates ), inventory )
198264 assert set (combined_candidates .values ()) == {1 }, combined_candidates
199265 assert combined_cases == baseline , (combined_cases , baseline )
266+ verify_manifests (results , ":test" )
200267 assert run ("single-shard" , shard_args (1 , count = 1 )) == baseline
201268
202269 other_baseline = run ("other-baseline" , task = "otherTest" )
@@ -218,7 +285,19 @@ def main():
218285 parser .add_argument ("--gradle" , default = "gradle" )
219286 parser .add_argument ("--junit-classpath" )
220287 parser .add_argument ("--work-dir" , type = Path )
288+ mode = parser .add_mutually_exclusive_group ()
289+ mode .add_argument ("--github-output" , type = Path )
290+ mode .add_argument ("--manifests" , type = Path )
291+ parser .add_argument ("--task" , help = "Expected Gradle task path when checking real manifests" )
221292 args = parser .parse_args ()
293+ if args .github_output :
294+ write_workflow_matrix (args .github_output )
295+ return
296+ if args .manifests :
297+ if not args .task :
298+ parser .error ("--manifests requires --task" )
299+ verify_manifests (args .manifests , args .task )
300+ return
222301 gradle = shutil .which (args .gradle )
223302 if not gradle :
224303 parser .error (f"Gradle executable not found: { args .gradle } " )
0 commit comments