Skip to content

Commit c52fbb8

Browse files
committed
test: cover a streamed double order with signed zeros and NaN
An outer hash join over a streamed side sorted by a double key with -0.0, 0.0, NaN and the infinities keeps its order, groups each key once, and returns Spark's rows for a top-k above it. The planner declares the order on the key normalized as the native sort normalizes it, and the top-k runs in a native plan of its own over the join's output.
1 parent 3533ca6 commit c52fbb8

1 file changed

Lines changed: 79 additions & 5 deletions

File tree

‎spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala‎

Lines changed: 79 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExcha
4444
import org.apache.spark.sql.execution.joins.{ShuffledHashJoinExec, SortMergeJoinExec}
4545
import org.apache.spark.sql.functions.{count, lit, sum}
4646
import org.apache.spark.sql.internal.SQLConf
47-
import org.apache.spark.sql.types.{ArrayType, IntegerType, LongType, MetadataBuilder, StructField, StructType}
47+
import org.apache.spark.sql.types.{ArrayType, DoubleType, IntegerType, LongType, MetadataBuilder, StructField, StructType}
4848

4949
import org.apache.comet.{CometConf, CometExplainInfo, ExtendedExplainInfo}
5050
import org.apache.comet.CometSparkSessionExtensions.{hasFallbackReason, isSpark35Plus}
@@ -2463,15 +2463,16 @@ class CometJoinSuite extends CometTestBase {
24632463
}
24642464
}
24652465

2466-
// Compares the rows of a fresh `df` as a multiset with the ones Spark returns for it. The rows
2467-
// of a key tie under the sort, so their order within it differs between the two.
2468-
private def sameRowsAsSpark(df: => DataFrame): Unit = {
2466+
// Compares the rows of a fresh `df` with the ones Spark returns for it, as a multiset unless
2467+
// `isSorted`. The rows of a key tie under a sort, so their order within it can differ between
2468+
// the two.
2469+
private def sameRowsAsSpark(df: => DataFrame, isSorted: Boolean = false): Unit = {
24692470
// withSQLConf returns Unit on Spark 3.x, so Spark's rows are assigned inside it.
24702471
var sparkRows = Seq.empty[Row]
24712472
withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
24722473
sparkRows = df.collect().toSeq
24732474
}
2474-
QueryTest.sameRows(sparkRows, df.collect().toSeq).foreach(fail(_))
2475+
QueryTest.sameRows(sparkRows, df.collect().toSeq, isSorted).foreach(fail(_))
24752476
}
24762477

24772478
test("outer hash joins keep the order of a native range") {
@@ -2608,6 +2609,79 @@ class CometJoinSuite extends CometTestBase {
26082609
}
26092610
}
26102611

2612+
test("outer hash joins keep a streamed double order with signed zeros and NaN") {
2613+
// Spark sorts -0.0 and 0.0 as equal and NaN above every other value. The planner declares the
2614+
// order on the key normalized as the native sort normalizes it, so an operator above the join
2615+
// compares keys as Spark does. A top-k above the join runs in a native plan of its own, over
2616+
// the join's output, so it cannot stop early on the declared order, and one sorting by the key
2617+
// alone reads only the first rows of each partition, which holds only if the join keeps them in
2618+
// order. A broadcast join, since Spark joins on the normalized key and a shuffled join would
2619+
// shuffle the streamed side again.
2620+
val keys =
2621+
Seq(Double.NegativeInfinity, -1.5, -0.0, 0.0, 1.5, Double.PositiveInfinity, Double.NaN)
2622+
withSQLConf(
2623+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
2624+
SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
2625+
CometConf.COMET_EXEC_IN_MEMORY_CACHE_ENABLED.key -> "false",
2626+
CometConf.COMET_CONVERT_FROM_IN_MEMORY_CACHE_ENABLED.key -> "true") {
2627+
// The build side holds -0.0 and not 0.0, so a streamed 0.0 matches only on the normalized
2628+
// key.
2629+
val buildData = Seq((Double.NegativeInfinity, 0), (-0.0, 1), (1.5, 2), (Double.NaN, 3))
2630+
withParquetTable((0 until 2000).map(i => (keys(i % keys.size), i)), "big") {
2631+
withParquetTable(buildData, "small") {
2632+
try {
2633+
val streamed =
2634+
spark.table("big").repartition(2, $"_2").sortWithinPartitions("_1").cache()
2635+
streamed.count()
2636+
val build = spark.table("small")
2637+
val order = SortOrder(BoundReference(0, DoubleType, nullable = true), Ascending)
2638+
Seq("left_outer", "right_outer").foreach { joinType =>
2639+
def joined: DataFrame = {
2640+
val hinted = build.hint("broadcast")
2641+
if (joinType == "left_outer") {
2642+
streamed.join(hinted, streamed("_1") === build("_1"), joinType)
2643+
} else {
2644+
hinted.join(streamed, build("_1") === streamed("_1"), joinType)
2645+
}
2646+
}
2647+
def rows: DataFrame =
2648+
joined.select(streamed("_1").as("k"), streamed("_2").as("v"), build("_2").as("w"))
2649+
val df = rows.sortWithinPartitions("k")
2650+
val plan = df.queryExecution.executedPlan
2651+
val context = s"$joinType\n$plan"
2652+
val joins = collect(plan) { case j: CometBroadcastHashJoinExec => j.nativeOp }
2653+
assert(joins.size == 1, context)
2654+
assert(joins.head.getHashJoin.getStreamedSortOrdersCount > 0, context)
2655+
assert(
2656+
collect(plan) { case s @ (_: CometSortExec | _: SortExec) => s }.isEmpty,
2657+
context)
2658+
sameRowsAsSpark(rows.sortWithinPartitions("k"))
2659+
assert(partitionsInOrder(df, order), context)
2660+
2661+
def agg: DataFrame = joined
2662+
.groupBy(streamed("_1").as("k"))
2663+
.agg(count(lit(1)), sum(streamed("_2")), count(build("_2")))
2664+
sameRowsAsSpark(agg)
2665+
val groupKeys = agg.collect().map(_.getDouble(0)).toSeq
2666+
assert(groupKeys.distinct.size == groupKeys.size, s"$joinType groups a key twice")
2667+
2668+
// The first 600 rows by the key end among the zeros, which tie, so which of them
2669+
// are taken depends on v alone. The first 300 descending take every NaN, which tie
2670+
// too, and the rest from the infinities. A limit on the key alone ends where the
2671+
// zeros start, since which of them it took would be arbitrary.
2672+
sameRowsAsSpark(rows.orderBy("k", "v").limit(600), isSorted = true)
2673+
sameRowsAsSpark(rows.orderBy($"k".desc, $"v").limit(300), isSorted = true)
2674+
val belowZero = (0 until 2000).count(i => keys(i % keys.size) < -1.0)
2675+
sameRowsAsSpark(rows.select("k").orderBy("k").limit(belowZero), isSorted = true)
2676+
}
2677+
} finally {
2678+
spark.catalog.clearCache()
2679+
}
2680+
}
2681+
}
2682+
}
2683+
}
2684+
26112685
// The setup several cache tests share: `big(_1, _2)` with 2000 rows over 40 sparse keys and
26122686
// `small` with a row for every even key, and `big` as `cached(k, v)`, partitioned on the key,
26132687
// sorted within each partition and cached, with 8-row batches and the native cache on. `views`

0 commit comments

Comments
 (0)