@@ -2175,18 +2175,21 @@ class CometIcebergWriteActionSuite
21752175 // https://github.com/apache/datafusion-comet/issues/6138. iceberg-rust compared float partition
21762176 // values with an equality that treated -0.0 and 0.0 as one value, so its writers filed both
21772177 // under whichever arrived first, and a read that pruned on the other value lost rows. Since
2178- // apache/iceberg-rust#3327 it keeps them apart, as iceberg-java does. One shuffle partition puts
2179- // every row in one task, so the zeros meet in one writer. The fanout writer gets them
2180- // interleaved. The clustered writer gets one row per partition: Spark's sort ties -0.0 with
2181- // 0.0, so a zero that repeats could come back unclustered, which iceberg-java rejects too.
2182- test(" signed-zero float and double identity partitions match iceberg-java" ) {
2178+ // apache/iceberg-rust#3327 it keeps them apart and puts every NaN in one partition, as
2179+ // iceberg-java does. One shuffle partition puts every row in one task, so the values meet in one
2180+ // writer. The fanout writer gets them interleaved. The clustered writer gets each partition's
2181+ // rows together: Spark's sort ties -0.0 with 0.0, so a zero that repeats could come back
2182+ // unclustered, which iceberg-java rejects too.
2183+ test(" signed-zero and NaN float and double identity partitions match iceberg-java" ) {
21832184 assumeNativeAcceleration()
21842185 withIcebergCatalog { warehouseDir =>
2185- val (negF, posF) = (" CAST('-0.0' AS FLOAT)" , " CAST(0.0 AS FLOAT)" )
2186- val (negD, posD) = (" CAST('-0.0' AS DOUBLE)" , " 0.0D" )
2187- val onePerPartition = Seq ((negF, posD), (posF, posD), (posF, negD), (negF, negD))
2188- val interleaved = onePerPartition ++ onePerPartition.reverse
2189- Seq (" true" -> interleaved, " false" -> onePerPartition).foreach { case (fanout, rows) =>
2186+ val (negF, posF, nanF) =
2187+ (" CAST('-0.0' AS FLOAT)" , " CAST(0.0 AS FLOAT)" , " CAST('NaN' AS FLOAT)" )
2188+ val (negD, posD, nanD) = (" CAST('-0.0' AS DOUBLE)" , " 0.0D" , " CAST('NaN' AS DOUBLE)" )
2189+ val zeros = Seq ((negF, posD), (posF, posD), (posF, negD), (negF, negD))
2190+ val nans = Seq ((nanF, nanD), (s " - $nanF" , s " - $nanD" ))
2191+ val interleaved = zeros ++ nans ++ zeros.reverse ++ nans
2192+ Seq (" true" -> interleaved, " false" -> (zeros ++ nans)).foreach { case (fanout, rows) =>
21902193 val (nativeTable, jvmTable) = (s " zero_native_ $fanout" , s " zero_jvm_ $fanout" )
21912194 Seq (nativeTable, jvmTable).foreach { table =>
21922195 spark.sql(s """
@@ -2207,31 +2210,30 @@ class CometIcebergWriteActionSuite
22072210
22082211 val dirs = partitionDirs(warehouseDir, nativeTable)
22092212 assert(dirs == partitionDirs(warehouseDir, jvmTable), s " fanout= $fanout: $dirs" )
2210- assert(dirs == Set (" f=-0.0/d=0.0" , " f=0.0/d=0.0" , " f=0.0/d=-0.0" , " f=-0.0/d=-0.0" ))
2211-
2212- // The committed partition values, not just their spelling in the path.
2213- def partitionValues (table : String ): Seq [String ] = spark
2214- .sql(s " SELECT CAST(partition AS STRING) FROM $catalog. $ns. $table.files " )
2215- .collect()
2216- .map(_.getString(0 ))
2217- .toSeq
2218- .sorted
22192213 assert(
2220- partitionValues(nativeTable) == partitionValues(jvmTable),
2221- s " fanout= $fanout: ${partitionValues(nativeTable)}" )
2214+ dirs == Set (
2215+ " f=-0.0/d=0.0" ,
2216+ " f=0.0/d=0.0" ,
2217+ " f=0.0/d=-0.0" ,
2218+ " f=-0.0/d=-0.0" ,
2219+ " f=NaN/d=NaN" ))
2220+ assert(
2221+ committedPartitions(nativeTable) == committedPartitions(jvmTable),
2222+ s " fanout= $fanout: ${committedPartitions(nativeTable)}" )
22222223
22232224 // Iceberg prunes on the partition value, so a merged partition changes what a read
22242225 // returns.
2225- Seq (s " f = $negF" , s " f = $posF" , s " d = $negD" , s " d = $posD" ).foreach { predicate =>
2226- def matching (table : String ): Seq [Int ] =
2227- spark
2228- .sql(s " SELECT id FROM $catalog. $ns. $table WHERE $predicate" )
2229- .collect()
2230- .map(_.getInt(0 ))
2231- .sorted
2232- .toSeq
2233- assert(matching(nativeTable) == matching(jvmTable), s " fanout= $fanout, $predicate" )
2234- }
2226+ Seq (s " f = $negF" , s " f = $posF" , s " d = $negD" , s " d = $posD" , s " d = $nanD" , " isnan(f)" )
2227+ .foreach { predicate =>
2228+ def matching (table : String ): Seq [Int ] =
2229+ spark
2230+ .sql(s " SELECT id FROM $catalog. $ns. $table WHERE $predicate" )
2231+ .collect()
2232+ .map(_.getInt(0 ))
2233+ .sorted
2234+ .toSeq
2235+ assert(matching(nativeTable) == matching(jvmTable), s " fanout= $fanout, $predicate" )
2236+ }
22352237 }
22362238 }
22372239 }
@@ -2272,6 +2274,39 @@ class CometIcebergWriteActionSuite
22722274 }
22732275 }
22742276
2277+ // Two partition fields of the same decimal type share one named Avro type, `decimal_10_2`.
2278+ // iceberg-rust defined it once per field, which the Avro specification forbids, until it moved
2279+ // to apache-avro 0.22 (apache/iceberg-rust#3354). iceberg-java reads each native task's manifest
2280+ // back, so this checks it still can. `decimal(38, 2)` covers the fixed-length encoding.
2281+ test(" two decimal identity partitions of the same type match iceberg-java" ) {
2282+ assumeNativeAcceleration()
2283+ withIcebergCatalog { warehouseDir =>
2284+ Seq (" 10, 2" , " 38, 2" ).foreach { decimalType =>
2285+ val suffix = decimalType.replace(" , " , " _" )
2286+ val (nativeTable, jvmTable) = (s " decimal_native_ $suffix" , s " decimal_jvm_ $suffix" )
2287+ Seq (nativeTable, jvmTable).foreach { table =>
2288+ spark.sql(s """
2289+ CREATE TABLE $catalog. $ns. $table (
2290+ id INT, d1 DECIMAL( $decimalType), d2 DECIMAL( $decimalType))
2291+ USING iceberg PARTITIONED BY (d1, d2)
2292+ """ )
2293+ }
2294+ val values = " (1, 123.45, -6.78), (2, 0.00, 1.10)"
2295+ assertNativeWriteEngages(nativeTable, Seq (1 , 2 )) {
2296+ spark.sql(s " INSERT INTO $catalog. $ns. $nativeTable VALUES $values" )
2297+ }
2298+ spark.sql(s " INSERT INTO $catalog. $ns. $jvmTable VALUES $values" )
2299+
2300+ val dirs = partitionDirs(warehouseDir, nativeTable)
2301+ assert(dirs == partitionDirs(warehouseDir, jvmTable), s " decimal( $decimalType): $dirs" )
2302+ assert(dirs == Set (" d1=123.45/d2=-6.78" , " d1=0.00/d2=1.10" ))
2303+ assert(
2304+ committedPartitions(nativeTable) == committedPartitions(jvmTable),
2305+ s " decimal( $decimalType): ${committedPartitions(nativeTable)}" )
2306+ }
2307+ }
2308+ }
2309+
22752310 // https://github.com/apache/datafusion-comet/issues/6145. iceberg-rust splits the calendar with
22762311 // `chrono`, which stops at year 262142, while a Spark date reaches year 5881580 and a timestamp
22772312 // year 294247. Past that, rendering a `days` or identity-date partition directory panicked the
@@ -4083,6 +4118,15 @@ class CometIcebergWriteActionSuite
40834118 * location. Comparing this set between a natively-written table and a JVM-written twin pins the
40844119 * on-disk layout against iceberg-java's `PartitionSpec#partitionToPath`.
40854120 */
4121+ /** Each data file's committed partition value and record count, sorted. */
4122+ private def committedPartitions (tableName : String ): Seq [String ] =
4123+ spark
4124+ .sql(s " SELECT CAST(partition AS STRING), record_count FROM $catalog. $ns. $tableName.files " )
4125+ .collect()
4126+ .map(_.toString)
4127+ .toSeq
4128+ .sorted
4129+
40864130 private def partitionDirs (warehouseDir : File , tableName : String ): Set [String ] = {
40874131 val location = warehouseDir.toURI.toString.stripSuffix(" /" )
40884132 spark
0 commit comments