Skip to content

Commit 913354c

Browse files
committed
fix(ci): verify complete Iceberg shard inventories
1 parent d6c3f6a commit 913354c

7 files changed

Lines changed: 295 additions & 13 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,9 @@ jobs:
8888
- name: Check micro benchmark runner
8989
run: python3 dev/ci/check-benchmark-runner.py
9090

91+
- name: Check Iceberg shard inventory validation
92+
run: python3 dev/ci/test-iceberg-shards.py
93+
9194
- name: Install actionlint
9295
run: |
9396
curl -sSfL https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash | bash

‎.github/workflows/iceberg_spark_test_reusable.yml‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,9 @@ jobs:
6363
build-native:
6464
name: Build Native Library
6565
runs-on: ubuntu-24.04
66+
outputs:
67+
shard-matrix: ${{ steps.shards.outputs.matrix }}
68+
shard-count: ${{ steps.shards.outputs.count }}
6669
container:
6770
image: amd64/rust
6871
steps:
@@ -74,6 +77,10 @@ jobs:
7477
rust-version: ${{ env.RUST_VERSION }}
7578
jdk-version: 17
7679

80+
- name: Define Iceberg test shards
81+
id: shards
82+
run: python3 dev/ci/check-iceberg-shards.py --github-output "$GITHUB_OUTPUT"
83+
7784
- name: Restore Cargo cache
7885
uses: actions/cache/restore@v6
7986
with:
@@ -113,8 +120,7 @@ jobs:
113120
needs: build-native
114121
strategy:
115122
fail-fast: false
116-
matrix:
117-
shard: [1, 2, 3, 4]
123+
matrix: ${{ fromJSON(needs.build-native.outputs.shard-matrix) }}
118124
name: iceberg-spark/iceberg-${{ inputs.iceberg-full }}/spark-${{ inputs.spark-full }}/scala-${{ inputs.scala }}/java-${{ inputs.java }}/shard-${{ matrix.shard }}
119125
runs-on: ubuntu-24.04
120126
container:
@@ -148,7 +154,7 @@ jobs:
148154
:iceberg-spark:iceberg-spark-${{ inputs.spark-short }}_${{ inputs.scala }}:test \
149155
--init-script ../dev/ci/iceberg-test-shards.gradle \
150156
-PcometShardTask=:iceberg-spark:iceberg-spark-${{ inputs.spark-short }}_${{ inputs.scala }}:test \
151-
-PcometShardIndex=${{ matrix.shard }} -PcometShardCount=${{ strategy.job-total }} \
157+
-PcometShardIndex=${{ matrix.shard }} -PcometShardCount=${{ needs.build-native.outputs.shard-count }} \
152158
-Pquick=true -x javadoc
153159
- name: Upload Iceberg shard inventory and test reports
154160
if: ${{ !cancelled() }}
@@ -160,6 +166,25 @@ jobs:
160166
apache-iceberg/**/build/test-results/test/*.xml
161167
retention-days: 7
162168

169+
iceberg-spark-shard-coverage:
170+
needs: iceberg-spark
171+
if: ${{ !cancelled() }}
172+
name: iceberg-spark-shard-coverage/iceberg-${{ inputs.iceberg-full }}/spark-${{ inputs.spark-full }}/scala-${{ inputs.scala }}/java-${{ inputs.java }}
173+
runs-on: ubuntu-slim
174+
steps:
175+
- uses: actions/checkout@v7
176+
- name: Download Iceberg shard inventories
177+
uses: actions/download-artifact@v8
178+
with:
179+
# Download all attempts: a failed-job rerun retains earlier successful
180+
# shards. The checker selects the latest inventory for each index.
181+
pattern: iceberg-spark-${{ inputs.iceberg-full }}-spark-${{ inputs.spark-full }}-scala-${{ inputs.scala }}-jdk${{ inputs.java }}-shard-*-attempt-*
182+
path: iceberg-shard-reports
183+
- name: Verify complete, disjoint Iceberg candidate coverage
184+
run: |
185+
python3 dev/ci/check-iceberg-shards.py --manifests iceberg-shard-reports \
186+
--task :iceberg-spark:iceberg-spark-${{ inputs.spark-short }}_${{ inputs.scala }}:test
187+
163188
iceberg-spark-extensions:
164189
needs: build-native
165190
name: iceberg-spark-extensions/iceberg-${{ inputs.iceberg-full }}/spark-${{ inputs.spark-full }}/scala-${{ inputs.scala }}/java-${{ inputs.java }}

‎dev/ci/check-iceberg-shards.py‎

Lines changed: 81 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
2323
The fixture downloads only JUnit. --junit-classpath accepts local JUnit jars for
2424
offline checks. --work-dir retains the fixture, logs, and comparison results.
25+
--github-output exports the workflow matrix/count from one shared definition.
26+
--manifests verifies the real inventories downloaded from an Iceberg CI run.
2527
"""
2628

2729
import argparse
@@ -36,6 +38,69 @@
3638

3739

3840
INIT_SCRIPT = Path(__file__).with_name("iceberg-test-shards.gradle")
41+
# The workflow matrix and Gradle's partition count must come from the same value,
42+
# not strategy.job-total (which includes any other matrix dimensions).
43+
SHARD_COUNT = 4
44+
45+
46+
def verify_manifests(root, task):
47+
"""Check each latest shard attempt against its independently captured baseline.
48+
49+
Rerunning only failed jobs leaves successful jobs' artifacts in an earlier
50+
attempt. Keep those, but never count two attempts of the same shard twice.
51+
Artifact download is scoped to one run and Iceberg/Spark/Scala/JDK tuple.
52+
"""
53+
attempts = {}
54+
for path in sorted(root.rglob("*.json")):
55+
manifest = json.loads(path.read_text())
56+
for key in ("shard", "count", "attempt"):
57+
if type(manifest.get(key)) is not int or manifest[key] < 1:
58+
raise ValueError(f"{path}: {key} must be a positive integer")
59+
key = manifest["shard"], manifest["attempt"]
60+
if key in attempts:
61+
raise ValueError(f"Duplicate inventory for shard/attempt {key}: {path}")
62+
attempts[key] = manifest
63+
64+
expected = set(range(1, SHARD_COUNT + 1))
65+
indices = {index for index, _ in attempts}
66+
if indices != expected:
67+
raise ValueError(f"Expected shard indices {sorted(expected)}, found {sorted(indices)}")
68+
69+
baseline = None
70+
combined = Counter()
71+
for index in sorted(expected):
72+
attempt = max(attempt for shard, attempt in attempts if shard == index)
73+
manifest = attempts[index, attempt]
74+
if manifest.get("task") != task or manifest["count"] != SHARD_COUNT:
75+
raise ValueError(f"Shard {index}: mismatched task or shard count")
76+
for field in ("candidates", "unshardedCandidates"):
77+
values = manifest.get(field)
78+
if not isinstance(values, list) or not all(isinstance(v, str) for v in values):
79+
raise ValueError(f"Shard {index}: {field} must be a list of class paths")
80+
if len(set(values)) != len(values):
81+
raise ValueError(f"Shard {index}: duplicate class paths in {field}")
82+
inventory = set(manifest["unshardedCandidates"])
83+
if not inventory:
84+
raise ValueError(f"Shard {index}: empty unsharded candidate inventory")
85+
if baseline is not None and baseline != inventory:
86+
raise ValueError(f"Shard {index}: unsharded inventories disagree")
87+
baseline = inventory
88+
combined.update(manifest["candidates"])
89+
90+
if set(combined) != baseline:
91+
missing, extra = baseline - set(combined), set(combined) - baseline
92+
raise ValueError(f"Shard coverage mismatch: missing={sorted(missing)}, extra={sorted(extra)}")
93+
duplicates = sorted(name for name, count in combined.items() if count != 1)
94+
if duplicates:
95+
raise ValueError(f"Candidate classes selected by multiple shards: {duplicates}")
96+
print(f"Verified {SHARD_COUNT} shards and {len(baseline)} candidate classes: "
97+
"the selected inventories equal the unsharded inventory exactly once.", flush=True)
98+
99+
100+
def write_workflow_matrix(output):
101+
with output.open("a") as stream:
102+
stream.write(f"matrix={json.dumps({'shard': list(range(1, SHARD_COUNT + 1))})}\n")
103+
stream.write(f"count={SHARD_COUNT}\n")
39104

40105

41106
def groovy_string(value):
@@ -168,7 +233,7 @@ def run(label, extra=(), task="test", expect_failure=False):
168233
print(f"{label}: {'expected failure' if expect_failure else 'passed'}", flush=True)
169234
return read_cases(root, task)
170235

171-
def shard_args(index, count=4, task=":test"):
236+
def shard_args(index, count=SHARD_COUNT, task=":test"):
172237
return ["--init-script", str(INIT_SCRIPT.resolve()), f"-PcometShardTask={task}",
173238
f"-PcometShardIndex={index}", f"-PcometShardCount={count}"]
174239

@@ -183,11 +248,12 @@ def shard_args(index, count=4, task=":test"):
183248
combined_cases = Counter()
184249
combined_candidates = Counter()
185250
failure_owner = None
186-
for index in range(1, 5):
251+
for index in range(1, SHARD_COUNT + 1):
187252
cases = run(f"shard-{index}", shard_args(index))
188253
manifest = json.loads((root / f"build/comet-shards/test-{index}.json").read_text())
189254
candidates = json.loads((root / "build/candidates.json").read_text())
190255
assert candidates == manifest["candidates"]
256+
assert set(manifest["unshardedCandidates"]) == inventory
191257
assert cases, f"empty fixture shard {index}"
192258
combined_cases.update(cases)
193259
combined_candidates.update(candidates)
@@ -197,6 +263,7 @@ def shard_args(index, count=4, task=":test"):
197263
assert set(combined_candidates) == inventory, (set(combined_candidates), inventory)
198264
assert set(combined_candidates.values()) == {1}, combined_candidates
199265
assert combined_cases == baseline, (combined_cases, baseline)
266+
verify_manifests(results, ":test")
200267
assert run("single-shard", shard_args(1, count=1)) == baseline
201268

202269
other_baseline = run("other-baseline", task="otherTest")
@@ -218,7 +285,19 @@ def main():
218285
parser.add_argument("--gradle", default="gradle")
219286
parser.add_argument("--junit-classpath")
220287
parser.add_argument("--work-dir", type=Path)
288+
mode = parser.add_mutually_exclusive_group()
289+
mode.add_argument("--github-output", type=Path)
290+
mode.add_argument("--manifests", type=Path)
291+
parser.add_argument("--task", help="Expected Gradle task path when checking real manifests")
221292
args = parser.parse_args()
293+
if args.github_output:
294+
write_workflow_matrix(args.github_output)
295+
return
296+
if args.manifests:
297+
if not args.task:
298+
parser.error("--manifests requires --task")
299+
verify_manifests(args.manifests, args.task)
300+
return
222301
gradle = shutil.which(args.gradle)
223302
if not gradle:
224303
parser.error(f"Gradle executable not found: {args.gradle}")

‎dev/ci/compute-changes.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,7 @@
195195
".github/actions/setup-iceberg-builder/**",
196196
"dev/ci/iceberg-test-shards.gradle",
197197
"dev/ci/check-iceberg-shards.py",
198+
"dev/ci/test-iceberg-shards.py",
198199
],
199200
"iceberg_1_9": [
200201
"native/**/src/**",
@@ -214,6 +215,7 @@
214215
".github/actions/setup-iceberg-builder/**",
215216
"dev/ci/iceberg-test-shards.gradle",
216217
"dev/ci/check-iceberg-shards.py",
218+
"dev/ci/test-iceberg-shards.py",
217219
],
218220
"iceberg_1_10": [
219221
"native/**/src/**",
@@ -233,6 +235,7 @@
233235
".github/actions/setup-iceberg-builder/**",
234236
"dev/ci/iceberg-test-shards.gradle",
235237
"dev/ci/check-iceberg-shards.py",
238+
"dev/ci/test-iceberg-shards.py",
236239
],
237240
"iceberg_1_11": [
238241
"native/**/src/**",
@@ -252,6 +255,7 @@
252255
".github/actions/setup-iceberg-builder/**",
253256
"dev/ci/iceberg-test-shards.gradle",
254257
"dev/ci/check-iceberg-shards.py",
258+
"dev/ci/test-iceberg-shards.py",
255259
],
256260
}
257261

‎dev/ci/iceberg-test-shards.gradle‎

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -66,27 +66,45 @@ gradle.projectsEvaluated {
6666
}
6767
testTask.inputs.property('cometShardIndex', shardIndex)
6868
testTask.inputs.property('cometShardCount', shardCount)
69+
int runAttempt = Integer.parseInt(System.getenv('GITHUB_RUN_ATTEMPT') ?: '1')
70+
testTask.inputs.property('cometShardRunAttempt', runAttempt)
6971
testTask.inputs.files(gradle.startParameter.initScripts).withPropertyName('cometShardScripts')
72+
boolean applyShardFilter = true
7073
testTask.exclude { FileTreeElement entry ->
71-
!entry.directory && entry.name.endsWith('.class') &&
74+
applyShardFilter && !entry.directory && entry.name.endsWith('.class') &&
7275
CometIcebergTestShard.owner(entry.name, shardCount) != shardIndex
7376
}
7477

7578
// Keep a machine-readable inventory next to the normal JUnit reports. This is
76-
// Gradle's final candidate set, so existing include/exclude rules still apply.
79+
// Gradle's candidate set, not the number of tests that JUnit actually executes.
7780
def manifest = testTask.project.layout.buildDirectory
7881
.file("comet-shards/${testTask.name}-${shardIndex}.json").get().asFile
7982
testTask.outputs.file(manifest)
8083
testTask.doFirst {
81-
def candidates = new TreeSet<String>()
82-
testTask.candidateClassFiles.visit { entry ->
83-
if (!entry.directory && entry.name.endsWith('.class')) {
84-
candidates.add(entry.relativePath.pathString)
84+
def inventory = {
85+
def candidates = new TreeSet<String>()
86+
testTask.candidateClassFiles.visit { entry ->
87+
if (!entry.directory && entry.name.endsWith('.class')) {
88+
candidates.add(entry.relativePath.pathString)
89+
}
8590
}
91+
candidates
8692
}
93+
// Compilation has completed by this point. Disable only our predicate while
94+
// recording the baseline, preserving even upstream exclusion closures. Then
95+
// restore it before collecting the selected candidates and executing tests.
96+
def unshardedCandidates
97+
applyShardFilter = false
98+
try {
99+
unshardedCandidates = inventory()
100+
} finally {
101+
applyShardFilter = true
102+
}
103+
def candidates = inventory()
87104
manifest.parentFile.mkdirs()
88105
manifest.text = JsonOutput.prettyPrint(JsonOutput.toJson([
89-
task: testTask.path, shard: shardIndex, count: shardCount, candidates: candidates])) + '\n'
106+
task: testTask.path, shard: shardIndex, count: shardCount, attempt: runAttempt,
107+
candidates: candidates, unshardedCandidates: unshardedCandidates])) + '\n'
90108
testTask.logger.lifecycle("Iceberg shard ${shardIndex}/${shardCount}: ${candidates.size()} candidate classes")
91109
}
92110
}

0 commit comments

Comments
 (0)