@@ -3712,10 +3712,10 @@ class CometInMemoryCacheSuite extends CometTestBase {
37123712
37133713 test(" a native window over a sorted cache declares the cache's order" ) {
37143714 // Spark plans no sort below a window over the cache's sort keys, so the scan has to declare
3715- // the order (#7223). A nullable key and a descending order key check the direction and null
3716- // placement. dense_rank runs in BoundedWindowAggExec and the UNBOUNDED FOLLOWING frame in
3717- // WindowAggExec. Within a key, v = k + 20 * j for j in 0 until 100, and a NULL key holds the
3718- // multiples of 20.
3715+ // the partition key's order (#7223). The key is nullable, so the null placement has to match,
3716+ // and the descending order key is not declared. dense_rank runs in BoundedWindowAggExec and
3717+ // the UNBOUNDED FOLLOWING frame in WindowAggExec. Within a key, v = k + 20 * j for j in
3718+ // 0 until 100, and a NULL key holds the multiples of 20.
37193719 val key = (v : Long ) => if (v % 20 == 0 ) null else v % 20
37203720 val queries = Seq (
37213721 " dense_rank() OVER (PARTITION BY k ORDER BY v DESC)" ->
@@ -3745,7 +3745,7 @@ class CometInMemoryCacheSuite extends CometTestBase {
37453745 checkWindows(
37463746 cached.selectExpr(" k" , " v" , s " $window AS w " ),
37473747 (0L until 2000L ).map(v => Row (key(v), v, expected(v))),
3748- Seq (Seq (" 0 Ascending NullsFirst" , " 1 Descending NullsLast " )))
3748+ Seq (Seq (" 0 Ascending NullsFirst" )))
37493749 }
37503750 }
37513751 } finally {
@@ -3836,13 +3836,51 @@ class CometInMemoryCacheSuite extends CometTestBase {
38363836 val plan = df.queryExecution.executedPlan
38373837 assert(collect(plan) { case w : CometWindowExec => w }.isEmpty, plan)
38383838 val reasons = new ExtendedExplainInfo ().getFallbackReasons(plan)
3839- assert(reasons.exists(_. contains(" the order of the window's input " ) ), reasons)
3839+ assert(reasons.contains(CometWindowExec .inexpressibleInputOrderReason ), reasons)
38403840 } finally {
38413841 spark.catalog.clearCache()
38423842 }
38433843 }
38443844 }
38453845
3846+ test(" a window over a cache whose order cannot be checked stays in Spark" ) {
3847+ // The filter keeps the serde from seeing that the outer join declared its order, so the
3848+ // window, reading that join's cache from the JVM, cannot check it. Declaring nothing would
3849+ // fail the native window.
3850+ withSQLConf(
3851+ SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> " false" ,
3852+ SQLConf .SHUFFLE_PARTITIONS .key -> " 2" ,
3853+ CometConf .COMET_EXEC_IN_MEMORY_CACHE_ENABLED .key -> " true" ,
3854+ CometConf .COMET_CONVERT_FROM_IN_MEMORY_CACHE_ENABLED .key -> " false" ) {
3855+ withParquetTable((0 until 2000 ).map(i => ((i % 40 ) * 1000 , i)), " big" ) {
3856+ withParquetTable((0 until 40 by 2 ).map(m => (m * 1000 , m)), " small" ) {
3857+ withTempView(" cached" , " joined" ) {
3858+ try {
3859+ spark
3860+ .table(" big" )
3861+ .toDF(" k" , " v" )
3862+ .repartition(2 , $" k" )
3863+ .sortWithinPartitions(" k" )
3864+ .cache()
3865+ .createOrReplaceTempView(" cached" )
3866+ spark.table(" cached" ).count()
3867+ sql(" SELECT /*+ BROADCAST(b) */ a.k, a.v, b._2 AS w FROM " +
3868+ " (SELECT * FROM cached WHERE v >= 0) a LEFT JOIN small b ON a.k = b._1" )
3869+ .cache()
3870+ .createOrReplaceTempView(" joined" )
3871+ spark.table(" joined" ).count()
3872+ checkSparkAnswerAndFallbackReason(
3873+ sql(" SELECT k, v, sum(v) OVER (PARTITION BY k) AS s FROM joined" ),
3874+ CometWindowExec .uncheckedInputOrderReason)
3875+ } finally {
3876+ spark.catalog.clearCache()
3877+ }
3878+ }
3879+ }
3880+ }
3881+ }
3882+ }
3883+
38463884 test(" an aggregate on the key above a hash join over a sorted cache groups each key once" ) {
38473885 // The order the join declares on its probe input reaches past the join, so the aggregate
38483886 // above it in the same native plan groups in sorted, streaming mode. That is right while
0 commit comments