Skip to content

Commit b4dd3d0

Browse files
committed
test: report streaming and Spark 3.4 CTAS/RTAS Iceberg writes, and count only each shard's latest attempt
The write report missed two kinds of Iceberg write that Spark runs itself: - A streaming micro-batch runs as `WriteToDataSourceV2Exec` over a `MicroBatchWrite`, not as a `V2ExistingTableWriteExec`. - On Spark 3.4, CTAS and RTAS write the table from the create or replace exec through `TableWriteExecHelper.writeWithV2`. Spark 3.5+ runs that write as a nested append or overwrite, which the listener already sees, so `IcebergTableAsSelectShim` recognizes the execs on 3.4 only. Both went unreported, so the Spark count and the total left them out and the native share read high. The summarizer took each shard's latest attempt from its report files, so a retry that recorded no writes was hidden behind the earlier attempt's records. It now takes the latest attempt from the shard artifact directories and names a shard whose latest attempt recorded nothing. `dev/ci/test-summarize-iceberg-writes.py` covers that and runs in Preflight.
1 parent 2a16295 commit b4dd3d0

9 files changed

Lines changed: 394 additions & 57 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,9 @@ jobs:
186186
- name: Check Iceberg shard inventory validation
187187
run: python3 dev/ci/test-iceberg-shards.py
188188

189+
- name: Check Iceberg write report summary
190+
run: python3 dev/ci/test-summarize-iceberg-writes.py
191+
189192
- name: Check CI config invariants
190193
run: python3 dev/ci/check-ci-config.py
191194

‎dev/ci/summarize-iceberg-writes.py‎

Lines changed: 42 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,11 @@
2727
2828
Files under a directory named like a shard artifact (...-shard-N-attempt-M)
2929
are counted only for the latest attempt of each shard, so a rerun of failed
30-
jobs does not count a shard twice. The summary is also appended to
31-
$GITHUB_STEP_SUMMARY when that is set. It never fails the job.
30+
jobs does not count a shard twice. The latest attempt is the newest such
31+
directory, whether or not it holds any report files, so a rerun that recorded
32+
no writes is reported as missing rather than replaced by an earlier attempt.
33+
The summary is also appended to $GITHUB_STEP_SUMMARY when that is set. It
34+
never fails the job.
3235
"""
3336

3437
import argparse
@@ -48,40 +51,59 @@
4851
TOP_REASONS = 20
4952

5053

54+
def shard_attempt(path):
55+
"""The (shard, attempt) of the shard artifact directory holding path, or None."""
56+
for part in path.parts:
57+
match = SHARD_ATTEMPT.search(part)
58+
if match:
59+
return int(match.group(1)), int(match.group(2))
60+
return None
61+
62+
5163
def report_files(roots):
52-
"""Every report file under roots, keeping only the latest attempt of each shard."""
64+
"""Every report file under roots, keeping only the latest attempt of each shard.
65+
66+
Returns the files and the (shard, attempt) pairs whose latest attempt holds no report file.
67+
The latest attempt comes from the artifact directories rather than the report files, because
68+
an attempt that recorded no writes still uploads its shard inventory and test reports.
69+
"""
5370
latest = {}
5471
files = []
55-
for root in roots:
56-
for path in sorted(Path(root).rglob("*.jsonl")):
57-
match = next(
58-
(SHARD_ATTEMPT.search(part) for part in path.parts if SHARD_ATTEMPT.search(part)),
59-
None,
60-
)
61-
if match:
62-
shard, attempt = int(match.group(1)), int(match.group(2))
63-
latest[shard] = max(latest.get(shard, 0), attempt)
64-
files.append((path, (shard, attempt)))
65-
else:
66-
files.append((path, None))
67-
return [path for path, key in files if key is None or latest[key[0]] == key[1]]
72+
for root in map(Path, roots):
73+
for path in [root, *sorted(root.rglob("*"))]:
74+
key = shard_attempt(path)
75+
if key:
76+
latest[key[0]] = max(latest.get(key[0], 0), key[1])
77+
if path.suffix == ".jsonl" and path.is_file():
78+
files.append((path, key))
79+
kept = [(path, key) for path, key in files if key is None or latest[key[0]] == key[1]]
80+
reported = {key for _, key in kept}
81+
missing = sorted(key for key in latest.items() if key not in reported)
82+
return [path for path, _ in kept], missing
6883

6984

7085
def load(roots):
86+
files, missing = report_files(roots)
7187
writes = []
72-
for path in report_files(roots):
88+
for path in files:
7389
for line in path.read_text(encoding="utf-8").splitlines():
7490
if line.strip():
7591
writes.append(json.loads(line))
76-
return writes
92+
return writes, missing
7793

7894

7995
def cell(text):
8096
return " ".join(text.split()).replace("|", "\\|")
8197

8298

83-
def summarize(title, writes):
99+
def summarize(title, writes, missing=()):
84100
lines = [f"### Iceberg writes: {title}", ""]
101+
for shard, attempt in missing:
102+
lines += [
103+
f"Shard {shard} recorded no Iceberg writes in its latest attempt ({attempt}), "
104+
"so none of its writes are counted below.",
105+
"",
106+
]
85107
if not writes:
86108
lines.append(
87109
"No Iceberg writes were recorded. Either the target ran none or "
@@ -137,7 +159,7 @@ def main():
137159
parser.add_argument("roots", nargs="+", help="directories holding the report files")
138160
args = parser.parse_args()
139161

140-
summary = summarize(args.title, load(args.roots))
162+
summary = summarize(args.title, *load(args.roots))
141163
print(summary)
142164
step_summary = os.environ.get("GITHUB_STEP_SUMMARY")
143165
if step_summary:
Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
#!/usr/bin/env python3
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
20+
"""Fast regression tests for the Iceberg write report summary."""
21+
22+
import importlib.util
23+
import json
24+
from pathlib import Path
25+
import tempfile
26+
import unittest
27+
28+
29+
SPEC = importlib.util.spec_from_file_location(
30+
"summarize_iceberg_writes", Path(__file__).with_name("summarize-iceberg-writes.py"))
31+
SUMMARIZE = importlib.util.module_from_spec(SPEC)
32+
SPEC.loader.exec_module(SUMMARIZE)
33+
34+
ARTIFACT = "iceberg-spark-1.11.0-spark-4.1.3-scala-2.13-jdk17-shard-{}-attempt-{}"
35+
36+
37+
def records(*writers):
38+
return "".join(
39+
json.dumps({"writer": w, "node": "AppendData", "reasons": [], "failed": False}) + "\n"
40+
for w in writers)
41+
42+
43+
class SummarizeIcebergWritesTest(unittest.TestCase):
44+
def setUp(self):
45+
self.temp = tempfile.TemporaryDirectory(prefix="comet-iceberg-writes-test-")
46+
self.addCleanup(self.temp.cleanup)
47+
self.root = Path(self.temp.name)
48+
49+
def attempt(self, shard, attempt, *writers):
50+
"""A shard attempt's artifact as the coverage job downloads it.
51+
52+
Every attempt uploads its test reports. Only an attempt that recorded writes has a
53+
report file.
54+
"""
55+
artifact = self.root / ARTIFACT.format(shard, attempt)
56+
reports = artifact / "build/test-results/test"
57+
reports.mkdir(parents=True)
58+
(reports / "TEST-org.example.TestFixture.xml").write_text("<testsuite/>")
59+
if writers:
60+
writes = artifact / "build/comet-iceberg-writes"
61+
writes.mkdir(parents=True)
62+
(writes / "iceberg-writes-fixture.jsonl").write_text(records(*writers))
63+
64+
def load(self):
65+
writes, missing = SUMMARIZE.load([self.root])
66+
return sorted(w["writer"] for w in writes), missing
67+
68+
def test_every_shard_counts_once(self):
69+
self.attempt(1, 1, "native")
70+
self.attempt(2, 1, "jvm", "spark")
71+
self.assertEqual(self.load(), (["jvm", "native", "spark"], []))
72+
73+
def test_latest_attempt_replaces_an_earlier_one(self):
74+
self.attempt(1, 1, "native", "native")
75+
self.attempt(1, 2, "jvm")
76+
self.assertEqual(self.load(), (["jvm"], []))
77+
78+
def test_retry_without_writes_is_reported_missing_instead_of_stale(self):
79+
self.attempt(1, 1, "native")
80+
self.attempt(1, 2)
81+
self.attempt(2, 1, "spark")
82+
self.assertEqual(self.load(), (["spark"], [(1, 2)]))
83+
summary = SUMMARIZE.summarize("fixture", *SUMMARIZE.load([self.root]))
84+
self.assertIn("Shard 1 recorded no Iceberg writes in its latest attempt (2)", summary)
85+
self.assertIn("| Comet native writer | 0 | 0.0% |", summary)
86+
self.assertIn("| Spark V2 write, not planned by Comet's split operator | 1 | 100.0% |",
87+
summary)
88+
89+
def test_root_that_is_itself_a_shard_artifact(self):
90+
self.attempt(1, 3)
91+
writes, missing = SUMMARIZE.load([self.root / ARTIFACT.format(1, 3)])
92+
self.assertEqual((writes, missing), ([], [(1, 3)]))
93+
94+
def test_files_outside_shard_artifacts_always_count(self):
95+
# A shard job and dev/local-ci.sh summarize their own report directory directly.
96+
(self.root / "iceberg-writes-local.jsonl").write_text(records("native", "jvm"))
97+
self.assertEqual(self.load(), (["jvm", "native"], []))
98+
99+
def test_no_writes_at_all(self):
100+
summary = SUMMARIZE.summarize("fixture", *SUMMARIZE.load([self.root / "missing"]))
101+
self.assertIn("No Iceberg writes were recorded", summary)
102+
103+
104+
if __name__ == "__main__":
105+
unittest.main()

‎docs/source/contributor-guide/iceberg-spark-tests.md‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -170,12 +170,14 @@ records one of three writers:
170170
- `native`: Comet's native writer (`CometIcebergWriteExec`).
171171
- `jvm`: Comet's split operator planned the write but kept Iceberg's JVM writer (`IcebergWriteExec`).
172172
The line includes the reasons Comet recorded for not converting it.
173-
- `spark`: Spark's own V2 write operator ran the write, for example `WriteDelta` for merge-on-read,
174-
so Comet's split operator never saw it.
173+
- `spark`: Spark's own V2 write operator ran the write, so Comet's split operator never saw it. Examples
174+
are `WriteDelta` for merge-on-read, `WriteToDataSourceV2` for a streaming micro-batch, and on Spark
175+
3.4 the CTAS and RTAS execs, which write the table themselves.
175176

176177
`dev/ci/summarize-iceberg-writes.py` turns these records into a table on the job's summary page. It
177178
shows the count and share of each writer, the most common fallback reasons, and the Spark write
178179
operators. Each shard and the extensions job gets its own table. The shard coverage job adds one for
179-
all shards together. The raw records are uploaded with the job's other reports. The summary never
180-
fails a job. `dev/local-ci.sh iceberg` prints the same summary after each shard and after the
181-
extensions target.
180+
all shards together, counting only the latest attempt of each shard. A shard whose latest attempt
181+
recorded no writes is named above the table rather than counted from an earlier attempt. The raw
182+
records are uploaded with the job's other reports. The summary never fails a job.
183+
`dev/local-ci.sh iceberg` prints the same summary after each shard and after the extensions target.

‎spark/src/main/scala/org/apache/comet/iceberg/IcebergWriteReportListener.scala‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,11 @@ import org.json4s.jackson.JsonMethods._
3232
import org.apache.spark.SparkConf
3333
import org.apache.spark.internal.Logging
3434
import org.apache.spark.sql.comet.{CometIcebergWriteExec, IcebergWriteExec}
35+
import org.apache.spark.sql.connector.write.BatchWrite
3536
import org.apache.spark.sql.execution.{CommandResultExec, QueryExecution, SparkPlan}
3637
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, QueryStageExec}
37-
import org.apache.spark.sql.execution.datasources.v2.V2ExistingTableWriteExec
38+
import org.apache.spark.sql.execution.datasources.v2.{V2ExistingTableWriteExec, WriteToDataSourceV2Exec}
39+
import org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite
3840
import org.apache.spark.sql.util.QueryExecutionListener
3941

4042
import org.apache.comet.CometConf.COMET_ICEBERG_WRITE_REPORT_DIR
@@ -111,9 +113,23 @@ object IcebergWriteReportListener {
111113
case w: IcebergWriteExec =>
112114
val reasons = w.getTagValue(CometExplainInfo.FALLBACK_REASONS).getOrElse(Set.empty)
113115
Seq(IcebergWrite(Jvm, w.nodeName, reasons.toSeq.sorted))
114-
case w: V2ExistingTableWriteExec
115-
if w.write.getClass.getName.startsWith("org.apache.iceberg.") =>
116+
case w: V2ExistingTableWriteExec if isIceberg(w.write) =>
117+
Seq(IcebergWrite(Spark, w.nodeName, Nil))
118+
// A streaming micro-batch, which Comet's split operator never plans.
119+
case w: WriteToDataSourceV2Exec if isIcebergMicroBatch(w.batchWrite) =>
120+
Seq(IcebergWrite(Spark, w.nodeName, Nil))
121+
// Spark 3.4 writes a CTAS or RTAS from the create or replace exec itself. Later versions run
122+
// that write as a nested append or overwrite, which this listener sees as a query of its own.
123+
case w if IcebergTableAsSelectShim.writeCatalog(w).exists(isIceberg) =>
116124
Seq(IcebergWrite(Spark, w.nodeName, Nil))
117125
case p => p.children.flatMap(writes)
118126
}
127+
128+
private def isIcebergMicroBatch(write: BatchWrite): Boolean = write match {
129+
case m: MicroBatchWrite => isIceberg(m.writeSupport)
130+
case _ => false
131+
}
132+
133+
private def isIceberg(obj: AnyRef): Boolean =
134+
obj.getClass.getName.startsWith("org.apache.iceberg.")
119135
}
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.comet.iceberg
21+
22+
import org.apache.spark.sql.connector.catalog.TableCatalog
23+
import org.apache.spark.sql.execution.SparkPlan
24+
import org.apache.spark.sql.execution.datasources.v2.{AtomicCreateTableAsSelectExec, AtomicReplaceTableAsSelectExec, CreateTableAsSelectExec, ReplaceTableAsSelectExec}
25+
26+
/**
27+
* Spark 3.4: CTAS and RTAS write the table from the create or replace exec itself, through
28+
* `TableWriteExecHelper.writeWithV2`, so the write never appears as a write node of its own.
29+
*/
30+
private[iceberg] object IcebergTableAsSelectShim {
31+
32+
/** The catalog `plan` writes through, when `plan` is a CTAS or RTAS exec. */
33+
def writeCatalog(plan: SparkPlan): Option[TableCatalog] = plan match {
34+
case p: CreateTableAsSelectExec => Some(p.catalog)
35+
case p: AtomicCreateTableAsSelectExec => Some(p.catalog)
36+
case p: ReplaceTableAsSelectExec => Some(p.catalog)
37+
case p: AtomicReplaceTableAsSelectExec => Some(p.catalog)
38+
case _ => None
39+
}
40+
}
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.comet.iceberg
21+
22+
import org.apache.spark.sql.connector.catalog.TableCatalog
23+
import org.apache.spark.sql.execution.SparkPlan
24+
25+
/**
26+
* Spark 3.5+: CTAS and RTAS run their write as a nested `AppendData` or `OverwriteByExpression`
27+
* query, which is planned and reported like any other write, so no create or replace exec writes
28+
* a table itself.
29+
*/
30+
private[iceberg] object IcebergTableAsSelectShim {
31+
32+
/** The catalog `plan` writes through, when `plan` is a CTAS or RTAS exec. */
33+
def writeCatalog(plan: SparkPlan): Option[TableCatalog] = None
34+
}
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.comet.iceberg
21+
22+
import org.apache.spark.sql.connector.catalog.TableCatalog
23+
import org.apache.spark.sql.execution.SparkPlan
24+
25+
/**
26+
* Spark 3.5+: CTAS and RTAS run their write as a nested `AppendData` or `OverwriteByExpression`
27+
* query, which is planned and reported like any other write, so no create or replace exec writes
28+
* a table itself.
29+
*/
30+
private[iceberg] object IcebergTableAsSelectShim {
31+
32+
/** The catalog `plan` writes through, when `plan` is a CTAS or RTAS exec. */
33+
def writeCatalog(plan: SparkPlan): Option[TableCatalog] = None
34+
}

0 commit comments

Comments
 (0)