@@ -3755,11 +3755,12 @@ class CometInMemoryCacheSuite extends CometTestBase {
37553755 }
37563756 }
37573757
3758- test(" stacked native windows over a sorted cache" ) {
3759- // Spark plans no sort between two windows that the cache's order satisfies, so the upper
3760- // window's input, which declares only the lower window's order, gets the rest declared next
3761- // to it (#7223). Alias a of sort key k shares its order. One partition, so no exchange
3762- // separates the windows.
3758+ test(" a native window declares its order next to the order its input declares" ) {
3759+ // Spark plans no sort above an operator that keeps the cache's order, so a window's input may
3760+ // declare only the order that operator needed: an upper window's input the lower window's,
3761+ // and an outer join's output its streamed side's. The window's order is declared next to it
3762+ // (#7223). Alias a of sort key k shares its order. One partition, so no exchange comes in
3763+ // between.
37633764 val ids = 0L until 24L
37643765 val key = (id : Long ) => if (id % 4 == 0 ) None else Some (id % 3 )
37653766 val nullable = (k : Option [Long ]) => k.map(Long .box).orNull
@@ -3802,6 +3803,18 @@ class CometInMemoryCacheSuite extends CometTestBase {
38023803 Row (nullable(k), nullable(k), id, keySums(k) * keyCounts(k))
38033804 },
38043805 Seq (Seq (" 1 Ascending NullsFirst" ), Seq (" 0 Ascending NullsFirst" )))
3806+ // The residual condition leaves rows of key 0 unmatched among matched ones, so the key
3807+ // stays together only if the join keeps its unmatched probe rows in place.
3808+ val one = spark.range(1 ).toDF(" j" ).hint(" broadcast" )
3809+ checkWindows(
3810+ cached
3811+ .join(one, col(" k" ) === col(" j" ) && col(" v" ) % 2 === 0 , " left_outer" )
3812+ .selectExpr(" k" , " a" , " v" , " sum(v) OVER (PARTITION BY a) AS s" ),
3813+ ids.map { id =>
3814+ val k = key(id)
3815+ Row (nullable(k), nullable(k), id, keySums(k))
3816+ },
3817+ Seq (Seq (" 1 Ascending NullsFirst" )))
38053818 } finally {
38063819 spark.catalog.clearCache()
38073820 }
@@ -3810,77 +3823,44 @@ class CometInMemoryCacheSuite extends CometTestBase {
38103823
38113824 test(" a window over a cache sorted by a collated string key stays in Spark" ) {
38123825 assume(isSpark40Plus, " collated string types require Spark 4.0+" )
3813- // The native side would declare a collated key in raw byte order, not Spark's collation
3814- // order, so the window falls back.
3826+ // The native window compares keys byte by byte, so it would split the UTF8_LCASE partition
3827+ // of '0a' and '0A' and rank them apart.
3828+ val key = (v : Long ) => s " ${v % 4 }${if (v % 8 < 4 ) " a" else " A" }"
38153829 withConversions(
38163830 SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> " false" ,
38173831 SQLConf .SHUFFLE_PARTITIONS .key -> " 2" ) {
38183832 try {
3819- val cached = spark
3833+ val source = spark
38203834 .range(200 )
38213835 .selectExpr(
38223836 " CAST(id % 4 AS STRING) || CASE WHEN id % 8 < 4 THEN 'a' ELSE 'A' END " +
38233837 " COLLATE UTF8_LCASE AS k" ,
3838+ " id % 2 AS n" ,
38243839 " id AS v" )
3825- .repartition(2 , col(" k" ))
3826- . sortWithinPartitions(" k " , " v " )
3827- .cache ()
3828- cached .count()
3840+ val byKey = source .repartition(2 , col(" k" )).sortWithinPartitions( " k " , " v " ).cache( )
3841+ val byGroup = source.repartition( 2 , col( " n " )). sortWithinPartitions(" n " , " k " ).cache( )
3842+ byKey.count ()
3843+ byGroup .count()
38293844
3830- val df = cached.selectExpr(" k" , " v" , " dense_rank() OVER (PARTITION BY k ORDER BY v) AS r" )
3831- // UTF8_LCASE puts 'a' and 'A' in one partition, so each of the four keys holds 50 rows.
3832- val rows = (0L until 200L ).map { v =>
3833- Row (s " ${v % 4 }${if (v % 8 < 4 ) " a" else " A" }" , v, (v / 4 + 1 ).toInt)
3845+ Seq (
3846+ // Each of the four keys holds 50 rows.
3847+ byKey.selectExpr(" k" , " v" , " dense_rank() OVER (PARTITION BY k ORDER BY v) AS r" ) ->
3848+ ((v : Long ) => Row (key(v), v, (v / 4 + 1 ).toInt)),
3849+ // Each group holds two keys, '0' and '2' or '1' and '3'.
3850+ byGroup.selectExpr(" n" , " k" , " dense_rank() OVER (PARTITION BY n ORDER BY k) AS r" ) ->
3851+ ((v : Long ) => Row (v % 2 , key(v), (v % 4 / 2 + 1 ).toInt))).foreach { case (df, row) =>
3852+ QueryTest .sameRows((0L until 200L ).map(row), df.collect().toSeq).foreach(fail(_))
3853+ val plan = df.queryExecution.executedPlan
3854+ assert(collect(plan) { case w : CometWindowExec => w }.isEmpty, plan)
3855+ val reasons = new ExtendedExplainInfo ().getFallbackReasons(plan)
3856+ assert(reasons.exists(_.startsWith(" Window: non-default string collation" )), reasons)
38343857 }
3835- QueryTest .sameRows(rows, df.collect().toSeq).foreach(fail(_))
3836- val plan = df.queryExecution.executedPlan
3837- assert(collect(plan) { case w : CometWindowExec => w }.isEmpty, plan)
3838- val reasons = new ExtendedExplainInfo ().getFallbackReasons(plan)
3839- assert(reasons.contains(CometWindowExec .inexpressibleInputOrderReason), reasons)
38403858 } finally {
38413859 spark.catalog.clearCache()
38423860 }
38433861 }
38443862 }
38453863
3846- test(" a window over a cache whose order cannot be checked stays in Spark" ) {
3847- // The filter keeps the serde from seeing that the outer join declared its order, so the
3848- // window, reading that join's cache from the JVM, cannot check it. Declaring nothing would
3849- // fail the native window.
3850- withSQLConf(
3851- SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> " false" ,
3852- SQLConf .SHUFFLE_PARTITIONS .key -> " 2" ,
3853- CometConf .COMET_EXEC_IN_MEMORY_CACHE_ENABLED .key -> " true" ,
3854- CometConf .COMET_CONVERT_FROM_IN_MEMORY_CACHE_ENABLED .key -> " false" ) {
3855- withParquetTable((0 until 2000 ).map(i => ((i % 40 ) * 1000 , i)), " big" ) {
3856- withParquetTable((0 until 40 by 2 ).map(m => (m * 1000 , m)), " small" ) {
3857- withTempView(" cached" , " joined" ) {
3858- try {
3859- spark
3860- .table(" big" )
3861- .toDF(" k" , " v" )
3862- .repartition(2 , $" k" )
3863- .sortWithinPartitions(" k" )
3864- .cache()
3865- .createOrReplaceTempView(" cached" )
3866- spark.table(" cached" ).count()
3867- sql(" SELECT /*+ BROADCAST(b) */ a.k, a.v, b._2 AS w FROM " +
3868- " (SELECT * FROM cached WHERE v >= 0) a LEFT JOIN small b ON a.k = b._1" )
3869- .cache()
3870- .createOrReplaceTempView(" joined" )
3871- spark.table(" joined" ).count()
3872- checkSparkAnswerAndFallbackReason(
3873- sql(" SELECT k, v, sum(v) OVER (PARTITION BY k) AS s FROM joined" ),
3874- CometWindowExec .uncheckedInputOrderReason)
3875- } finally {
3876- spark.catalog.clearCache()
3877- }
3878- }
3879- }
3880- }
3881- }
3882- }
3883-
38843864 test(" an aggregate on the key above a hash join over a sorted cache groups each key once" ) {
38853865 // The order the join declares on its probe input reaches past the join, so the aggregate
38863866 // above it in the same native plan groups in sorted, streaming mode. That is right while
0 commit comments