Skip to content

Commit f8db9b5

Browse files
committed
test: keep multi-byte strings and Long.MinValue out of the partitioned write test
The CI containers run the JVM with an ASCII platform charset, so raw multi-byte partition directory names written by the pinned iceberg-rust cannot be reopened by iceberg-java for the metrics rebuild; partition on the long column instead. iceberg-rust's truncate transform also overflows on Long.MinValue in debug builds, so that boundary row is not written.
1 parent 49868b6 commit f8db9b5

1 file changed

Lines changed: 13 additions & 8 deletions

File tree

‎spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -171,14 +171,21 @@ class CometIcebergSystemFunctionSuite
171171
withSourceTable {
172172
val table = s"$catalog.db.hidden_partitioning"
173173
// No `write.distribution-mode`: Iceberg picks hash distribution for a partitioned table,
174-
// which plans a shuffle and a local sort on the partition transforms.
174+
// which plans a shuffle and a local sort on the partition transforms. The string column is
175+
// deliberately not a partition source: its multi-byte values would land in partition
176+
// directory names, which iceberg-java reads back through the JVM's platform charset.
175177
sql(s"""
176-
CREATE TABLE $table (i32 INT, str STRING, ts TIMESTAMP, dt DATE)
178+
CREATE TABLE $table (i32 INT, i64 BIGINT, ts TIMESTAMP, dt DATE)
177179
USING iceberg
178-
PARTITIONED BY (bucket(4, i32), truncate(2, str), days(ts), months(dt))""")
180+
PARTITIONED BY (bucket(4, i32), truncate(1000, i64), days(ts), months(dt))""")
181+
// iceberg-rust's own truncate transform, which the writer uses for partition values, does
182+
// `v - ((v % w) + w) % w` without wrapping and overflows on Long.MinValue in debug builds
183+
// (Java wraps), so the boundary row stays out of the written set.
184+
val rows =
185+
s"SELECT i32, i64, ts, dt FROM $source WHERE i64 IS NULL OR i64 <> ${Long.MinValue}"
179186
try {
180187
val plans = capturePlans(spark) {
181-
sql(s"INSERT INTO $table SELECT i32, str, ts, dt FROM $source")
188+
sql(s"INSERT INTO $table $rows")
182189
}
183190
val writePlans = plans.filter(plan =>
184191
collectWithSubqueries(plan) { case w: CometIcebergWriteExec => w }.nonEmpty)
@@ -196,16 +203,14 @@ class CometIcebergSystemFunctionSuite
196203
s"the distribution shuffle stayed on Spark:\n$plan")
197204
}
198205

199-
checkAnswer(
200-
sql(s"SELECT i32, str, ts, dt FROM $table"),
201-
sql(s"SELECT i32, str, ts, dt FROM $source").collect())
206+
checkAnswer(sql(s"SELECT i32, i64, ts, dt FROM $table"), sql(rows).collect())
202207

203208
// Iceberg's own view of the partitions must match what the JVM transforms compute over
204209
// the written rows: one partition per distinct transform tuple.
205210
withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
206211
val expected = sql(s"""
207212
SELECT COUNT(*) FROM (
208-
SELECT DISTINCT $catalog.system.bucket(4, i32), $catalog.system.truncate(2, str),
213+
SELECT DISTINCT $catalog.system.bucket(4, i32), $catalog.system.truncate(1000, i64),
209214
$catalog.system.days(ts), $catalog.system.months(dt)
210215
FROM $table)""").collect().head.getLong(0)
211216
val actual = sql(s"SELECT COUNT(*) FROM $table.partitions").collect().head.getLong(0)

0 commit comments

Comments
 (0)