@@ -44,7 +44,7 @@ import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExcha
4444import org .apache .spark .sql .execution .joins .{ShuffledHashJoinExec , SortMergeJoinExec }
4545import org .apache .spark .sql .functions .{count , lit , sum }
4646import org .apache .spark .sql .internal .SQLConf
47- import org .apache .spark .sql .types .{ArrayType , IntegerType , LongType , MetadataBuilder , StructField , StructType }
47+ import org .apache .spark .sql .types .{ArrayType , DoubleType , IntegerType , LongType , MetadataBuilder , StructField , StructType }
4848
4949import org .apache .comet .{CometConf , CometExplainInfo , ExtendedExplainInfo }
5050import org .apache .comet .CometSparkSessionExtensions .{hasFallbackReason , isSpark35Plus }
@@ -2463,15 +2463,16 @@ class CometJoinSuite extends CometTestBase {
24632463 }
24642464 }
24652465
2466- // Compares the rows of a fresh `df` as a multiset with the ones Spark returns for it. The rows
2467- // of a key tie under the sort, so their order within it differs between the two.
2468- private def sameRowsAsSpark (df : => DataFrame ): Unit = {
2466+ // Compares the rows of a fresh `df` with the ones Spark returns for it, as a multiset unless
2467+ // `isSorted`. The rows of a key tie under a sort, so their order within it can differ between
2468+ // the two.
2469+ private def sameRowsAsSpark (df : => DataFrame , isSorted : Boolean = false ): Unit = {
24692470 // withSQLConf returns Unit on Spark 3.x, so Spark's rows are assigned inside it.
24702471 var sparkRows = Seq .empty[Row ]
24712472 withSQLConf(CometConf .COMET_ENABLED .key -> " false" ) {
24722473 sparkRows = df.collect().toSeq
24732474 }
2474- QueryTest .sameRows(sparkRows, df.collect().toSeq).foreach(fail(_))
2475+ QueryTest .sameRows(sparkRows, df.collect().toSeq, isSorted ).foreach(fail(_))
24752476 }
24762477
24772478 test(" outer hash joins keep the order of a native range" ) {
@@ -2608,6 +2609,79 @@ class CometJoinSuite extends CometTestBase {
26082609 }
26092610 }
26102611
2612+ test(" outer hash joins keep a streamed double order with signed zeros and NaN" ) {
2613+ // Spark sorts -0.0 and 0.0 as equal and NaN above every other value. The planner declares the
2614+ // order on the key normalized as the native sort normalizes it, so an operator above the join
2615+ // compares keys as Spark does. A top-k above the join runs in a native plan of its own, over
2616+ // the join's output, so it cannot stop early on the declared order, and one sorting by the key
2617+ // alone reads only the first rows of each partition, which holds only if the join keeps them in
2618+ // order. A broadcast join, since Spark joins on the normalized key and a shuffled join would
2619+ // shuffle the streamed side again.
2620+ val keys =
2621+ Seq (Double .NegativeInfinity , - 1.5 , - 0.0 , 0.0 , 1.5 , Double .PositiveInfinity , Double .NaN )
2622+ withSQLConf(
2623+ SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> " false" ,
2624+ SQLConf .AUTO_BROADCASTJOIN_THRESHOLD .key -> " -1" ,
2625+ CometConf .COMET_EXEC_IN_MEMORY_CACHE_ENABLED .key -> " false" ,
2626+ CometConf .COMET_CONVERT_FROM_IN_MEMORY_CACHE_ENABLED .key -> " true" ) {
2627+ // The build side holds -0.0 and not 0.0, so a streamed 0.0 matches only on the normalized
2628+ // key.
2629+ val buildData = Seq ((Double .NegativeInfinity , 0 ), (- 0.0 , 1 ), (1.5 , 2 ), (Double .NaN , 3 ))
2630+ withParquetTable((0 until 2000 ).map(i => (keys(i % keys.size), i)), " big" ) {
2631+ withParquetTable(buildData, " small" ) {
2632+ try {
2633+ val streamed =
2634+ spark.table(" big" ).repartition(2 , $" _2" ).sortWithinPartitions(" _1" ).cache()
2635+ streamed.count()
2636+ val build = spark.table(" small" )
2637+ val order = SortOrder (BoundReference (0 , DoubleType , nullable = true ), Ascending )
2638+ Seq (" left_outer" , " right_outer" ).foreach { joinType =>
2639+ def joined : DataFrame = {
2640+ val hinted = build.hint(" broadcast" )
2641+ if (joinType == " left_outer" ) {
2642+ streamed.join(hinted, streamed(" _1" ) === build(" _1" ), joinType)
2643+ } else {
2644+ hinted.join(streamed, build(" _1" ) === streamed(" _1" ), joinType)
2645+ }
2646+ }
2647+ def rows : DataFrame =
2648+ joined.select(streamed(" _1" ).as(" k" ), streamed(" _2" ).as(" v" ), build(" _2" ).as(" w" ))
2649+ val df = rows.sortWithinPartitions(" k" )
2650+ val plan = df.queryExecution.executedPlan
2651+ val context = s " $joinType\n $plan"
2652+ val joins = collect(plan) { case j : CometBroadcastHashJoinExec => j.nativeOp }
2653+ assert(joins.size == 1 , context)
2654+ assert(joins.head.getHashJoin.getStreamedSortOrdersCount > 0 , context)
2655+ assert(
2656+ collect(plan) { case s @ (_ : CometSortExec | _ : SortExec ) => s }.isEmpty,
2657+ context)
2658+ sameRowsAsSpark(rows.sortWithinPartitions(" k" ))
2659+ assert(partitionsInOrder(df, order), context)
2660+
2661+ def agg : DataFrame = joined
2662+ .groupBy(streamed(" _1" ).as(" k" ))
2663+ .agg(count(lit(1 )), sum(streamed(" _2" )), count(build(" _2" )))
2664+ sameRowsAsSpark(agg)
2665+ val groupKeys = agg.collect().map(_.getDouble(0 )).toSeq
2666+ assert(groupKeys.distinct.size == groupKeys.size, s " $joinType groups a key twice " )
2667+
2668+ // The first 600 rows by the key end among the zeros, which tie, so which of them
2669+ // are taken depends on v alone. The first 300 descending take every NaN, which tie
2670+ // too, and the rest from the infinities. A limit on the key alone ends where the
2671+ // zeros start, since which of them it took would be arbitrary.
2672+ sameRowsAsSpark(rows.orderBy(" k" , " v" ).limit(600 ), isSorted = true )
2673+ sameRowsAsSpark(rows.orderBy($" k" .desc, $" v" ).limit(300 ), isSorted = true )
2674+ val belowZero = (0 until 2000 ).count(i => keys(i % keys.size) < - 1.0 )
2675+ sameRowsAsSpark(rows.select(" k" ).orderBy(" k" ).limit(belowZero), isSorted = true )
2676+ }
2677+ } finally {
2678+ spark.catalog.clearCache()
2679+ }
2680+ }
2681+ }
2682+ }
2683+ }
2684+
26112685 // The setup several cache tests share: `big(_1, _2)` with 2000 rows over 40 sparse keys and
26122686 // `small` with a row for every even key, and `big` as `cached(k, v)`, partitioned on the key,
26132687 // sorted within each partition and cached, with 8-row batches and the native cache on. `views`
0 commit comments