|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one |
| 3 | + * or more contributor license agreements. See the NOTICE file |
| 4 | + * distributed with this work for additional information |
| 5 | + * regarding copyright ownership. The ASF licenses this file |
| 6 | + * to you under the Apache License, Version 2.0 (the |
| 7 | + * "License"); you may not use this file except in compliance |
| 8 | + * with the License. You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, |
| 13 | + * software distributed under the License is distributed on an |
| 14 | + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | + * KIND, either express or implied. See the License for the |
| 16 | + * specific language governing permissions and limitations |
| 17 | + * under the License. |
| 18 | + */ |
| 19 | + |
| 20 | +package org.apache.comet.contrib.delta |
| 21 | + |
| 22 | +import org.apache.spark.sql.DataFrame |
| 23 | +import org.apache.spark.sql.comet.{CometDeltaNativeScanExec, CometDeltaScanMarker, CometScanExec} |
| 24 | +import org.apache.spark.sql.execution.FileSourceScanExec |
| 25 | +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec |
| 26 | +import org.apache.spark.sql.functions.col |
| 27 | + |
| 28 | +// Investigates StatsCollectionSuite "gather stats", which fails under Comet with |
| 29 | +// `java.util.NoSuchElementException: None.get` from `recordsScanned`. That helper locates |
| 30 | +// the scan in `executedPlan` and reads its `numOutputRows`. Two things broke it under Comet: |
| 31 | +// |
| 32 | +// 1. (native bug) `CometDeltaNativeScanExec` never recorded output rows -- it streamed |
| 33 | +// batches with no BaselineMetrics, so `numOutputRows` was always 0. Fixed in |
| 34 | +// contrib/delta/native/src/kernel_scan.rs (BaselineMetrics.record_output per batch). |
| 35 | +// |
| 36 | +// 2. (plan-shape) the Delta scan appears in DIFFERENT shapes depending on whether AQE |
| 37 | +// wrapped the plan, which gates Comet's marker->native conversion (it runs in AQE |
| 38 | +// query-stage prep): |
| 39 | +// - AQE on (this base forces it): CometDeltaNativeScanExec (converted); |
| 40 | +// - AQE off (Delta's own suites disable it for deterministic plans): the plan keeps |
| 41 | +// a CometDeltaScanMarker -- a LeafExecNode that delegates execution to the wrapped |
| 42 | +// FileSourceScanExec (vanilla Delta read), so its rows live on `originalScan`. |
| 43 | +// The suite's pre-execution `.find` also missed the scan entirely under AQE. |
| 44 | +// |
| 45 | +// This suite proves the data-skipping behaviour is correct in BOTH shapes by reading the |
| 46 | +// scan's numOutputRows the same robust way the diff patches `recordsScanned` to: materialize |
| 47 | +// first, strip AQE, then read the metric off whichever scan node the plan carries. |
| 48 | +class CometDeltaStatsSkippingReproSuite extends CometDeltaTestBase { |
| 49 | + |
| 50 | + private def numOutputRowsOfScan(df: DataFrame): Long = { |
| 51 | + // Materialize ONCE: finalizes the plan AND populates the executed scan's metric. Do not |
| 52 | + // execute the scan node again -- the native metric accumulates and would double-count. |
| 53 | + df.collect() |
| 54 | + val plan = df.queryExecution.executedPlan match { |
| 55 | + case a: AdaptiveSparkPlanExec => a.executedPlan |
| 56 | + case p => p |
| 57 | + } |
| 58 | + val metrics = plan.collectFirst { |
| 59 | + case s: CometDeltaNativeScanExec => s.metrics // AQE on: converted native scan |
| 60 | + case m: CometDeltaScanMarker => m.originalScan.metrics // AQE off: marker -> wrapped scan |
| 61 | + case s: CometScanExec => s.metrics |
| 62 | + case f: FileSourceScanExec => f.metrics |
| 63 | + }.getOrElse(fail(s"no Delta scan node found in finalized plan:\n$plan")) |
| 64 | + metrics("numOutputRows").value |
| 65 | + } |
| 66 | + |
| 67 | + private def writeGatherStatsTable(path: String): Unit = { |
| 68 | + // Exact mirror of StatsCollectionSuite "gather stats": 9 rows (1..9) across 10 partitions |
| 69 | + // (~1 row per file), partitioned by `odd = id % 2 == 1`. |
| 70 | + spark |
| 71 | + .range(1, 10, 1, 10) |
| 72 | + .withColumn("odd", col("id") % 2 === 1) |
| 73 | + .write |
| 74 | + .partitionBy("odd") |
| 75 | + .format("delta") |
| 76 | + .save(path) |
| 77 | + } |
| 78 | + |
| 79 | + private def checkSkipping(): Unit = { |
| 80 | + withTempDir { dir => |
| 81 | + val path = dir.getAbsolutePath |
| 82 | + writeGatherStatsTable(path) |
| 83 | + val df = spark.read.format("delta").load(path) |
| 84 | + // Disable parquet row-level pushdown exactly as the suite does, so the only thing that |
| 85 | + // can reduce the scan's output below the full row set is Delta data skipping. |
| 86 | + withSQLConf("spark.sql.parquet.filterPushdown" -> "false") { |
| 87 | + assert(numOutputRowsOfScan(df) == 9, "unfiltered read should scan all 9 rows") |
| 88 | + assert( |
| 89 | + numOutputRowsOfScan(df.where("id = 1")) == 1, |
| 90 | + "id = 1 should data-skip to the single file whose min/max bracket 1") |
| 91 | + } |
| 92 | + } |
| 93 | + } |
| 94 | + |
| 95 | + test("data skipping parity with AQE on (marker converted to native scan)") { |
| 96 | + assume(deltaSparkAvailable, "delta-spark not on the test classpath; skipping") |
| 97 | + withSQLConf("spark.sql.adaptive.enabled" -> "true") { |
| 98 | + checkSkipping() |
| 99 | + } |
| 100 | + } |
| 101 | + |
| 102 | + test("data skipping parity with AQE off (CometDeltaScanMarker fallback)") { |
| 103 | + assume(deltaSparkAvailable, "delta-spark not on the test classpath; skipping") |
| 104 | + // Reproduce Delta's own-suite environment, where AQE is off so the marker is left in the |
| 105 | + // plan and delegates to the vanilla FileSourceScanExec. |
| 106 | + withSQLConf("spark.sql.adaptive.enabled" -> "false") { |
| 107 | + checkSkipping() |
| 108 | + } |
| 109 | + } |
| 110 | +} |
0 commit comments