@@ -3692,6 +3692,13 @@ class CometInMemoryCacheSuite extends CometTestBase {
36923692 }
36933693 }
36943694
3695+ // The order a native window declares on its input, one "index direction nulls" per key.
3696+ private def windowInputOrder (window : CometWindowExec ): Seq [String ] =
3697+ window.nativeOp.getWindow.getInputSortOrdersList.asScala.toSeq.map { expr =>
3698+ val order = expr.getSortOrder
3699+ s " ${order.getChild.getBound.getIndex} ${order.getDirection} ${order.getNullOrdering}"
3700+ }
3701+
36953702 test(" a native window over a sorted cache declares the cache's order" ) {
36963703 // Spark plans no sort below a window over the cache's sort keys, and the native window
36973704 // rejected the scan, which declares no order, with "Expects PARTITION BY expression to be
@@ -3731,7 +3738,10 @@ class CometInMemoryCacheSuite extends CometTestBase {
37313738 withClue(s " nativeCache= $nativeCache, $window\n $plan\n " ) {
37323739 val windows = collect(plan) { case w : CometWindowExec => w }
37333740 assert(windows.size == 1 )
3734- assert(windows.head.nativeOp.getWindow.getInputSortOrdersCount == 2 )
3741+ assert(
3742+ windowInputOrder(windows.head) == Seq (
3743+ " 0 Ascending NullsFirst" ,
3744+ " 1 Descending NullsLast" ))
37353745 assert(collect(plan) { case s @ (_ : CometSortExec | _ : SortExec ) => s }.isEmpty)
37363746 }
37373747 }
@@ -3742,6 +3752,52 @@ class CometInMemoryCacheSuite extends CometTestBase {
37423752 }
37433753 }
37443754
3755+ test(" stacked native windows over a sorted cache keep the cache's whole order" ) {
3756+ // The lower window needs only k ordered and the upper one k and v. Spark plans no sort
3757+ // between them, since the lower window reports the cache's order, so the lower window has to
3758+ // declare all of it for the upper one to start (#7223). Within a key, v = k + 20 * j for j
3759+ // in 0 until 100, and a NULL key holds the multiples of 20, so each key sums to
3760+ // 100 * k + 99000.
3761+ val key = (v : Long ) => if (v % 20 == 0 ) null else v % 20
3762+ Seq (true , false ).foreach { nativeCache =>
3763+ withConversions(
3764+ SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> " false" ,
3765+ SQLConf .AUTO_BROADCASTJOIN_THRESHOLD .key -> " -1" ,
3766+ SQLConf .SHUFFLE_PARTITIONS .key -> " 2" ,
3767+ CometConf .COMET_EXEC_IN_MEMORY_CACHE_ENABLED .key -> nativeCache.toString) {
3768+ try {
3769+ val cached = spark
3770+ .range(2000 )
3771+ .selectExpr(" CASE WHEN id % 20 = 0 THEN NULL ELSE id % 20 END AS k" , " id AS v" )
3772+ .repartition(2 , col(" k" ))
3773+ .sortWithinPartitions(" k" , " v" )
3774+ .cache()
3775+ cached.count()
3776+
3777+ val df = cached
3778+ .selectExpr(" k" , " v" , " sum(v) OVER (PARTITION BY k) AS s" )
3779+ .selectExpr(" k" , " v" , " sum(s) OVER (PARTITION BY k, v) AS t" )
3780+ val rows = (0L until 2000L ).map(v => Row (key(v), v, 100 * (v % 20 ) + 99000 ))
3781+ QueryTest .sameRows(rows, df.collect().toSeq).foreach(fail(_))
3782+ val plan = df.queryExecution.executedPlan
3783+ withClue(s " nativeCache= $nativeCache\n $plan\n " ) {
3784+ val windows = collect(plan) { case w : CometWindowExec => w }
3785+ assert(windows.size == 2 )
3786+ windows.foreach { window =>
3787+ assert(
3788+ windowInputOrder(window) == Seq (
3789+ " 0 Ascending NullsFirst" ,
3790+ " 1 Ascending NullsFirst" ))
3791+ }
3792+ assert(collect(plan) { case s @ (_ : CometSortExec | _ : SortExec ) => s }.isEmpty)
3793+ }
3794+ } finally {
3795+ spark.catalog.clearCache()
3796+ }
3797+ }
3798+ }
3799+ }
3800+
37453801 test(" a window over a cache sorted by a collated string key stays in Spark" ) {
37463802 assume(isSpark40Plus, " collated string types require Spark 4.0+" )
37473803 // The native side would declare the order of a collated key in raw byte order, which is not
0 commit comments