@@ -21,7 +21,7 @@ package org.apache.comet.exec
2121
2222import scala .jdk .CollectionConverters ._
2323
24- import org .apache .spark .sql .CometTestBase
24+ import org .apache .spark .sql .{ CometTestBase , Row }
2525import org .apache .spark .sql .comet .{CometLocalTopKExec , CometNativeScanExec , CometTakeOrderedAndProjectExec }
2626import org .apache .spark .sql .execution .{LocalTableScanExec , SparkPlan }
2727import org .apache .spark .sql .internal .SQLConf
@@ -40,9 +40,10 @@ class CometTopKSuite extends CometTestBase {
4040 private def operators (op : Operator ): Seq [Operator ] =
4141 Seq (op) ++ op.getChildrenList.asScala.toSeq.flatMap(operators)
4242
43- for (partitions <- Seq (1 , 3 ); adaptive <- Seq (false , true )) {
44- test(s " local TopK shares the native scan: partitions= $partitions, AQE= $adaptive" ) {
43+ for (partitions <- Seq (1 , 3 ); adaptive <- Seq (false , true ); enabled <- Seq ( false , true ) ) {
44+ test(s " local TopK shares the scan: partitions= $partitions, AQE= $adaptive, filter= $enabled " ) {
4545 withSQLConf(
46+ CometConf .COMET_EXEC_TOPK_DYNAMIC_FILTER_ENABLED .key -> enabled.toString,
4647 SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> adaptive.toString,
4748 SQLConf .LEAF_NODE_DEFAULT_PARALLELISM .key -> partitions.toString,
4849 SQLConf .FILES_MAX_PARTITION_BYTES .key -> " 4194304" ) {
@@ -58,6 +59,7 @@ class CometTopKSuite extends CometTestBase {
5859 Seq (classOf [CometLocalTopKExec ], classOf [CometNativeScanExec ]))
5960 val local = localTopK(plan)
6061 assert(local.limit == 12 )
62+ assert(local.dynamicFilterEnabled == enabled)
6163 assert(local.executeColumnar().getNumPartitions == partitions)
6264 val block = Operator .parseFrom(local.serializedPlanOpt.plan.get)
6365 assert(block.hasSort && block.getSort.getFetch == 12 )
@@ -78,7 +80,9 @@ class CometTopKSuite extends CometTestBase {
7880 }
7981
8082 test(" local TopK preserves nulls, ties and small inputs" ) {
81- withSQLConf(SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> " false" ) {
83+ withSQLConf(
84+ SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> " false" ,
85+ CometConf .COMET_EXEC_TOPK_DYNAMIC_FILTER_ENABLED .key -> " true" ) {
8286 withTempPath { path =>
8387 spark
8488 .range(0 , 24 , 1 , 2 )
@@ -113,6 +117,114 @@ class CometTopKSuite extends CometTestBase {
113117 }
114118 }
115119
120+ for (pageIndex <- Seq (false , true ); rowFilter <- Seq (false , true )) {
121+ test(s " TopK prunes its Spark Parquet reader: pageIndex= $pageIndex, rowFilter= $rowFilter" ) {
122+ withSQLConf(
123+ SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> " false" ,
124+ SQLConf .LEAF_NODE_DEFAULT_PARALLELISM .key -> " 1" ,
125+ CometConf .COMET_BATCH_SIZE .key -> " 100" ,
126+ CometConf .COMET_RESPECT_DATAFUSION_CONFIGS .key -> " true" ,
127+ " spark.comet.datafusion.execution.parquet.enable_page_index" -> pageIndex.toString,
128+ CometConf .COMET_PARQUET_ROW_FILTER_PUSHDOWN_ENABLED .key -> rowFilter.toString) {
129+ withTempDir { dir =>
130+ val path = new org.apache.hadoop.fs.Path (dir.toURI.toString, " topk-groups.parquet" )
131+ val schema = org.apache.parquet.schema.MessageTypeParser .parseMessageType("""
132+ |message root {
133+ | required int64 payload;
134+ | required int32 k;
135+ |}
136+ |""" .stripMargin)
137+ val writer = createParquetWriter(schema, path, rowGroupSize = 1L )
138+ try {
139+ // Three improvements are needed: 209 -> 109 -> 9. The intervening
140+ // groups become prunable only after the reader starts producing batches.
141+ Seq (
142+ 200 until 300 ,
143+ 100 until 200 ,
144+ 150 until 250 ,
145+ 0 until 100 ,
146+ 50 until 150 ,
147+ 300 until 400 ).flatten.foreach { key =>
148+ val row = new org.apache.parquet.example.data.simple.SimpleGroup (schema)
149+ row.add(0 , key.toLong)
150+ row.add(1 , key)
151+ writer.write(row)
152+ }
153+ } finally {
154+ writer.close()
155+ }
156+ val footer = org.apache.parquet.hadoop.ParquetFileReader .open(
157+ org.apache.parquet.hadoop.util.HadoopInputFile
158+ .fromPath(path, spark.sessionState.newHadoopConf()))
159+ try {
160+ assert(
161+ footer.getFooter.getBlocks.asScala.map(_.getRowCount).toSeq == Seq .fill(6 )(100L ))
162+ } finally {
163+ footer.close()
164+ }
165+ withParquetTable(path.toString, " topk_groups" ) {
166+ val scans = Seq .newBuilder[Map [String , Long ]]
167+ Seq (false , true ).foreach { enabled =>
168+ withSQLConf(
169+ CometConf .COMET_EXEC_TOPK_DYNAMIC_FILTER_ENABLED .key -> enabled.toString) {
170+ // Project the second physical column to exercise reader predicate remapping.
171+ val query = sql(" SELECT k FROM topk_groups ORDER BY k LIMIT 10" )
172+ val (_, plan) =
173+ checkSparkAnswerAndOperator(query, Seq (classOf [CometLocalTopKExec ]))
174+ val local = localTopK(plan)
175+ assert(local.metrics(" output_rows" ).value == 10L )
176+ assert(local.metrics(" dynamic_filter_reader_filters_attached" ).value ==
177+ (if (enabled) 1L else 0L ))
178+ assert(! local.metrics.contains(" bytes_scanned" ))
179+ val scan = collect(plan) { case scan : CometNativeScanExec => scan }.head
180+ val metrics = scan.metrics.map { case (name, metric) => name -> metric.value }
181+ // checkSparkAnswerAndOperator executes a copy. Execute this DataFrame
182+ // twice as well, so the same Spark plan cannot retain a heap threshold.
183+ val expected = (0 until 10 ).map(Row (_))
184+ checkAnswer(query, expected)
185+ checkAnswer(query, expected)
186+ scans += metrics
187+ }
188+ }
189+ val Seq (disabled, enabled) = scans.result()
190+ assert(disabled(" output_rows" ) == 600L )
191+ assert(disabled(" row_groups_pruned_dynamic_filter" ) == 0L )
192+ if (! pageIndex && ! rowFilter) {
193+ assert(enabled(" row_groups_pruned_dynamic_filter" ) >= 3L )
194+ assert(enabled(" output_rows" ) <= 300L )
195+ assert(enabled(" bytes_scanned" ) < disabled(" bytes_scanned" ))
196+ }
197+ }
198+ }
199+ }
200+ }
201+ }
202+
203+ test(" TopK filtering preserves ordered inputs and empty scans" ) {
204+ withSQLConf(
205+ CometConf .COMET_EXEC_TOPK_DYNAMIC_FILTER_ENABLED .key -> " true" ,
206+ SQLConf .OPTIMIZER_EXCLUDED_RULES .key ->
207+ " org.apache.spark.sql.catalyst.optimizer.EliminateSorts" ) {
208+ withParquetTable((0 until 20 ).map(i => (i, 20 - i)), " topk_input" ) {
209+ val query = spark
210+ .table(" topk_input" )
211+ .sortWithinPartitions(" _1" , " _2" )
212+ .orderBy(" _1" )
213+ .limit(5 )
214+ val (_, plan) = checkSparkAnswerAndOperator(query)
215+ val topK = collect(plan) { case topK : CometTakeOrderedAndProjectExec => topK }.head
216+ assert(topK.orderingSatisfies)
217+ assert(collect(plan) { case local : CometLocalTopKExec => local }.isEmpty)
218+ }
219+ withTempPath { path =>
220+ spark.range(0 ).write.parquet(path.getCanonicalPath)
221+ withParquetTable(path.getCanonicalPath, " topk_empty" ) {
222+ checkSparkAnswerAndOperator(sql(" SELECT id FROM topk_empty ORDER BY id LIMIT 10" ))
223+ }
224+ }
225+ }
226+ }
227+
116228 test(" TopK filtering is opt-in and belongs to the local native plan" ) {
117229 assert(! CometConf .COMET_EXEC_TOPK_DYNAMIC_FILTER_ENABLED .get())
118230 withParquetTable((0 until 100 ).map(i => Tuple1 (i)), " topk_input" ) {
@@ -185,7 +297,7 @@ class CometTopKSuite extends CometTestBase {
185297 val query = sql(" SELECT k, ts FROM topk_overflow ORDER BY k LIMIT 1" )
186298 assert(localTopK(query.queryExecution.executedPlan).dynamicFilterEnabled == enabled)
187299 val (sparkError, cometError) = checkSparkAnswerMaybeThrows(query)
188- assert(sparkError.exists(isOverflow), s " Spark must reject overflowing timestamps " )
300+ assert(sparkError.exists(isOverflow), " Spark must reject overflowing timestamps" )
189301 assert(
190302 cometError.exists(isOverflow),
191303 s " filtering= $enabled must preserve Spark's overflow error; Comet: $cometError" )
0 commit comments