Skip to content

Commit 1036ae5

Browse files
committed
test: cover NaN and same-typed decimal identity partitions
The lifted gate also admits NaN partition values, so the signed-zero test writes NaN and -NaN through both writers too. The bump changes how iceberg-rust writes a named Avro type that two partition fields share, so a new test writes two decimal identity partitions of the same type, in both encodings, and compares them with iceberg-java. The detection test now uses plain tuples. Scala 2.12 flagged the adapted argument list as an error under -Pstrict-warnings.
1 parent 5374888 commit 1036ae5

2 files changed

Lines changed: 79 additions & 35 deletions

File tree

‎spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala‎

Lines changed: 75 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -2175,18 +2175,21 @@ class CometIcebergWriteActionSuite
21752175
// https://github.com/apache/datafusion-comet/issues/6138. iceberg-rust compared float partition
21762176
// values with an equality that treated -0.0 and 0.0 as one value, so its writers filed both
21772177
// under whichever arrived first, and a read that pruned on the other value lost rows. Since
2178-
// apache/iceberg-rust#3327 it keeps them apart, as iceberg-java does. One shuffle partition puts
2179-
// every row in one task, so the zeros meet in one writer. The fanout writer gets them
2180-
// interleaved. The clustered writer gets one row per partition: Spark's sort ties -0.0 with
2181-
// 0.0, so a zero that repeats could come back unclustered, which iceberg-java rejects too.
2182-
test("signed-zero float and double identity partitions match iceberg-java") {
2178+
// apache/iceberg-rust#3327 it keeps them apart and puts every NaN in one partition, as
2179+
// iceberg-java does. One shuffle partition puts every row in one task, so the values meet in one
2180+
// writer. The fanout writer gets them interleaved. The clustered writer gets each partition's
2181+
// rows together: Spark's sort ties -0.0 with 0.0, so a zero that repeats could come back
2182+
// unclustered, which iceberg-java rejects too.
2183+
test("signed-zero and NaN float and double identity partitions match iceberg-java") {
21832184
assumeNativeAcceleration()
21842185
withIcebergCatalog { warehouseDir =>
2185-
val (negF, posF) = ("CAST('-0.0' AS FLOAT)", "CAST(0.0 AS FLOAT)")
2186-
val (negD, posD) = ("CAST('-0.0' AS DOUBLE)", "0.0D")
2187-
val onePerPartition = Seq((negF, posD), (posF, posD), (posF, negD), (negF, negD))
2188-
val interleaved = onePerPartition ++ onePerPartition.reverse
2189-
Seq("true" -> interleaved, "false" -> onePerPartition).foreach { case (fanout, rows) =>
2186+
val (negF, posF, nanF) =
2187+
("CAST('-0.0' AS FLOAT)", "CAST(0.0 AS FLOAT)", "CAST('NaN' AS FLOAT)")
2188+
val (negD, posD, nanD) = ("CAST('-0.0' AS DOUBLE)", "0.0D", "CAST('NaN' AS DOUBLE)")
2189+
val zeros = Seq((negF, posD), (posF, posD), (posF, negD), (negF, negD))
2190+
val nans = Seq((nanF, nanD), (s"-$nanF", s"-$nanD"))
2191+
val interleaved = zeros ++ nans ++ zeros.reverse ++ nans
2192+
Seq("true" -> interleaved, "false" -> (zeros ++ nans)).foreach { case (fanout, rows) =>
21902193
val (nativeTable, jvmTable) = (s"zero_native_$fanout", s"zero_jvm_$fanout")
21912194
Seq(nativeTable, jvmTable).foreach { table =>
21922195
spark.sql(s"""
@@ -2207,31 +2210,30 @@ class CometIcebergWriteActionSuite
22072210

22082211
val dirs = partitionDirs(warehouseDir, nativeTable)
22092212
assert(dirs == partitionDirs(warehouseDir, jvmTable), s"fanout=$fanout: $dirs")
2210-
assert(dirs == Set("f=-0.0/d=0.0", "f=0.0/d=0.0", "f=0.0/d=-0.0", "f=-0.0/d=-0.0"))
2211-
2212-
// The committed partition values, not just their spelling in the path.
2213-
def partitionValues(table: String): Seq[String] = spark
2214-
.sql(s"SELECT CAST(partition AS STRING) FROM $catalog.$ns.$table.files")
2215-
.collect()
2216-
.map(_.getString(0))
2217-
.toSeq
2218-
.sorted
22192213
assert(
2220-
partitionValues(nativeTable) == partitionValues(jvmTable),
2221-
s"fanout=$fanout: ${partitionValues(nativeTable)}")
2214+
dirs == Set(
2215+
"f=-0.0/d=0.0",
2216+
"f=0.0/d=0.0",
2217+
"f=0.0/d=-0.0",
2218+
"f=-0.0/d=-0.0",
2219+
"f=NaN/d=NaN"))
2220+
assert(
2221+
committedPartitions(nativeTable) == committedPartitions(jvmTable),
2222+
s"fanout=$fanout: ${committedPartitions(nativeTable)}")
22222223

22232224
// Iceberg prunes on the partition value, so a merged partition changes what a read
22242225
// returns.
2225-
Seq(s"f = $negF", s"f = $posF", s"d = $negD", s"d = $posD").foreach { predicate =>
2226-
def matching(table: String): Seq[Int] =
2227-
spark
2228-
.sql(s"SELECT id FROM $catalog.$ns.$table WHERE $predicate")
2229-
.collect()
2230-
.map(_.getInt(0))
2231-
.sorted
2232-
.toSeq
2233-
assert(matching(nativeTable) == matching(jvmTable), s"fanout=$fanout, $predicate")
2234-
}
2226+
Seq(s"f = $negF", s"f = $posF", s"d = $negD", s"d = $posD", s"d = $nanD", "isnan(f)")
2227+
.foreach { predicate =>
2228+
def matching(table: String): Seq[Int] =
2229+
spark
2230+
.sql(s"SELECT id FROM $catalog.$ns.$table WHERE $predicate")
2231+
.collect()
2232+
.map(_.getInt(0))
2233+
.sorted
2234+
.toSeq
2235+
assert(matching(nativeTable) == matching(jvmTable), s"fanout=$fanout, $predicate")
2236+
}
22352237
}
22362238
}
22372239
}
@@ -2272,6 +2274,39 @@ class CometIcebergWriteActionSuite
22722274
}
22732275
}
22742276

2277+
// Two partition fields of the same decimal type share one named Avro type, `decimal_10_2`.
2278+
// iceberg-rust defined it once per field, which the Avro specification forbids, until it moved
2279+
// to apache-avro 0.22 (apache/iceberg-rust#3354). iceberg-java reads each native task's manifest
2280+
// back, so this checks it still can. `decimal(38, 2)` covers the fixed-length encoding.
2281+
test("two decimal identity partitions of the same type match iceberg-java") {
2282+
assumeNativeAcceleration()
2283+
withIcebergCatalog { warehouseDir =>
2284+
Seq("10, 2", "38, 2").foreach { decimalType =>
2285+
val suffix = decimalType.replace(", ", "_")
2286+
val (nativeTable, jvmTable) = (s"decimal_native_$suffix", s"decimal_jvm_$suffix")
2287+
Seq(nativeTable, jvmTable).foreach { table =>
2288+
spark.sql(s"""
2289+
CREATE TABLE $catalog.$ns.$table (
2290+
id INT, d1 DECIMAL($decimalType), d2 DECIMAL($decimalType))
2291+
USING iceberg PARTITIONED BY (d1, d2)
2292+
""")
2293+
}
2294+
val values = "(1, 123.45, -6.78), (2, 0.00, 1.10)"
2295+
assertNativeWriteEngages(nativeTable, Seq(1, 2)) {
2296+
spark.sql(s"INSERT INTO $catalog.$ns.$nativeTable VALUES $values")
2297+
}
2298+
spark.sql(s"INSERT INTO $catalog.$ns.$jvmTable VALUES $values")
2299+
2300+
val dirs = partitionDirs(warehouseDir, nativeTable)
2301+
assert(dirs == partitionDirs(warehouseDir, jvmTable), s"decimal($decimalType): $dirs")
2302+
assert(dirs == Set("d1=123.45/d2=-6.78", "d1=0.00/d2=1.10"))
2303+
assert(
2304+
committedPartitions(nativeTable) == committedPartitions(jvmTable),
2305+
s"decimal($decimalType): ${committedPartitions(nativeTable)}")
2306+
}
2307+
}
2308+
}
2309+
22752310
// https://github.com/apache/datafusion-comet/issues/6145. iceberg-rust splits the calendar with
22762311
// `chrono`, which stops at year 262142, while a Spark date reaches year 5881580 and a timestamp
22772312
// year 294247. Past that, rendering a `days` or identity-date partition directory panicked the
@@ -4083,6 +4118,15 @@ class CometIcebergWriteActionSuite
40834118
* location. Comparing this set between a natively-written table and a JVM-written twin pins the
40844119
* on-disk layout against iceberg-java's `PartitionSpec#partitionToPath`.
40854120
*/
4121+
/** Each data file's committed partition value and record count, sorted. */
4122+
private def committedPartitions(tableName: String): Seq[String] =
4123+
spark
4124+
.sql(s"SELECT CAST(partition AS STRING), record_count FROM $catalog.$ns.$tableName.files")
4125+
.collect()
4126+
.map(_.toString)
4127+
.toSeq
4128+
.sorted
4129+
40864130
private def partitionDirs(warehouseDir: File, tableName: String): Set[String] = {
40874131
val location = warehouseDir.toURI.toString.stripSuffix("/")
40884132
spark

‎spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1506,10 +1506,10 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes
15061506
// apart as iceberg-java does. A nested source field resolves the same way.
15071507
withDetectionCatalog { _ =>
15081508
Seq(
1509-
"part_float" -> ("v FLOAT", "v", "CAST(1.5 AS FLOAT)"),
1510-
"part_double" -> ("v DOUBLE", "v", "1.5D"),
1511-
"part_nested" -> ("s STRUCT<v: DOUBLE>", "s.v", "named_struct('v', 1.5D)")).foreach {
1512-
case (table, (column, partitionField, value)) =>
1509+
("part_float", "v FLOAT", "v", "CAST(1.5 AS FLOAT)"),
1510+
("part_double", "v DOUBLE", "v", "1.5D"),
1511+
("part_nested", "s STRUCT<v: DOUBLE>", "s.v", "named_struct('v', 1.5D)")).foreach {
1512+
case (table, column, partitionField, value) =>
15131513
spark.sql(s"""
15141514
CREATE TABLE $catalog.$ns.$table (id INT, $column)
15151515
USING iceberg PARTITIONED BY ($partitionField)

0 commit comments

Comments
 (0)