Skip to content

Commit 204440a

Browse files
committed
fix: declare the order of a sorted bucketed table read by the native scan
With spark.sql.legacy.bucketedTableScan.outputOrdering on, Spark reports a sorted bucketed table's sort columns as the scan's order when every bucket has one file. The serde trusted that order and sent it, but the planner declared an order only over a JVM input, so an outer hash join over the native scan moved its unmatched rows after Spark had dropped the sort above it. A scan of files reads the files of its one file group one after the other, so the planner now counts it as a source whose order holds.
1 parent b50b2cd commit 204440a

2 files changed

Lines changed: 112 additions & 14 deletions

File tree

‎native/core/src/execution/planner.rs‎

Lines changed: 47 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,8 @@ use datafusion_comet_spark_expr::{
9797
CsvWriteOptions, EvalMode, ListPositionsExpr, SparkArraysZipFunc, SparkBloomFilterVersion,
9898
SparkListAgg, SparkPercentile, Subquery, SumInteger, ToCsv,
9999
};
100+
use datafusion_datasource::file_scan_config::FileScanConfig;
101+
use datafusion_datasource::source::DataSourceExec;
100102
use datafusion_datasource::TableSchema;
101103
use iceberg::expr::Bind;
102104

@@ -3080,23 +3082,25 @@ impl PhysicalPlanner {
30803082

30813083
/// `plan`, declaring `sort_orders` as the order its rows arrive in, unless it already
30823084
/// declares one. Spark's plan knows the order of an input that enters the native plan from
3083-
/// the JVM, but the `ScanExec` it arrives through declares none. Only that order is declared:
3084-
/// when an operator in between may have moved rows, `plan` is left as it is.
3085+
/// the JVM, and of a scan of sorted files, but the `ScanExec` or the file scan the rows
3086+
/// arrive through declares none. Only that order is declared: when an operator in between may
3087+
/// have moved rows, `plan` is left as it is.
30853088
///
30863089
/// The serde sends an order only when the streamed side's Comet operators report it, and it
30873090
/// checks that nothing under them moves rows (`mayReportFalseOrder`), while this walk asks
30883091
/// the native operators the same thing through `maintains_input_order`. The two agree as
3089-
/// long as a Comet operator that reports an order keeps it natively and says so. One that
3090-
/// does not lands here with `plan` unchanged, and the join then moves its unmatched rows
3091-
/// after Spark has dropped the sort above it. Nothing here can fall back at that point, so
3092-
/// the invariant is kept at the operator: see "Adding a New Operator" in the contributor
3093-
/// guide.
3092+
/// long as a Comet operator that reports an order keeps it natively and says so, and a Comet
3093+
/// leaf that reports one is a source this walk knows or declares its order itself, as the
3094+
/// native range does. One that does not lands here with `plan` unchanged, and the join then
3095+
/// moves its unmatched rows after Spark has dropped the sort above it. Nothing here can fall
3096+
/// back at that point, so the invariant is kept at the operator: see "Adding a New Operator"
3097+
/// in the contributor guide.
30943098
fn with_declared_order(
30953099
&self,
30963100
plan: Arc<dyn ExecutionPlan>,
30973101
sort_orders: &[Expr],
30983102
) -> Result<Arc<dyn ExecutionPlan>, ExecutionError> {
3099-
if plan.output_ordering().is_some() || !Self::keeps_jvm_input_order(&plan) {
3103+
if plan.output_ordering().is_some() || !Self::keeps_source_order(&plan) {
31003104
return Ok(plan);
31013105
}
31023106
let exprs = sort_orders
@@ -3109,14 +3113,23 @@ impl PhysicalPlanner {
31093113
}
31103114
}
31113115

3112-
/// Whether `plan`'s rows come from an input outside the native plan through operators that
3113-
/// keep their input's order, so that the order Spark reports for them still holds. Any other
3114-
/// operator may have moved rows, such as the null-aware anti join, whose streamed side Comet
3115-
/// builds on, and declaring Spark's order over it would be wrong.
3116-
fn keeps_jvm_input_order(plan: &Arc<dyn ExecutionPlan>) -> bool {
3116+
/// Whether `plan`'s rows come from a source whose order Spark knows through operators that
3117+
/// keep their input's order, so that the order Spark reports for them still holds. A source is
3118+
/// an input from the JVM, or a scan of files, which reads the files of its one file group one
3119+
/// after the other: Spark reports an order for a file scan only when every partition reads one
3120+
/// sorted file, as for a sorted bucketed table with one file per bucket. Any other operator may
3121+
/// have moved rows, such as the null-aware anti join, whose streamed side Comet builds on, and
3122+
/// declaring Spark's order over it would be wrong.
3123+
fn keeps_source_order(plan: &Arc<dyn ExecutionPlan>) -> bool {
31173124
if plan.is::<ScanExec>() || plan.is::<ShuffleScanExec>() {
31183125
return true;
31193126
}
3127+
if let Some(scan) = plan.downcast_ref::<DataSourceExec>() {
3128+
return scan
3129+
.data_source()
3130+
.downcast_ref::<FileScanConfig>()
3131+
.is_some_and(|config| config.file_groups.len() == 1);
3132+
}
31203133
// DataFusion reports every right hash join as keeping its probe order, but a right outer
31213134
// join keeps it only when the probe declares one, and no input on this path declares an
31223135
// order. An inner join emits its matches probe row by probe row, and a right semi, anti or
@@ -3139,7 +3152,7 @@ impl PhysicalPlanner {
31393152
.zip(plan.maintains_input_order())
31403153
.filter_map(|(child, keeps)| keeps.then_some(child));
31413154
match (kept.next(), kept.next()) {
3142-
(Some(child), None) => Self::keeps_jvm_input_order(child),
3155+
(Some(child), None) => Self::keeps_source_order(child),
31433156
_ => false,
31443157
}
31453158
}
@@ -6564,6 +6577,26 @@ mod tests {
65646577
);
65656578
}
65666579

6580+
#[test]
6581+
fn declared_order_reaches_a_scan_of_one_file_group() {
6582+
// A scan of files reads the files of its one file group one after the other, so the order
6583+
// Spark reports for it, a sorted bucketed table's, holds. With more than one file group,
6584+
// DataFusion may hand a file to another partition.
6585+
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));
6586+
let scan = |groups: usize| -> Arc<dyn ExecutionPlan> {
6587+
let source = Arc::new(ParquetSource::new(Arc::clone(&schema))) as Arc<dyn FileSource>;
6588+
let file_groups = (0..groups)
6589+
.map(|i| FileGroup::new(vec![PartitionedFile::new(format!("f{i}.parquet"), 8)]))
6590+
.collect();
6591+
let config = FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), source)
6592+
.with_file_groups(file_groups)
6593+
.build();
6594+
Arc::new(DataSourceExec::new(Arc::new(config)))
6595+
};
6596+
assert!(PhysicalPlanner::keeps_source_order(&scan(1)));
6597+
assert!(!PhysicalPlanner::keeps_source_order(&scan(2)));
6598+
}
6599+
65676600
#[tokio::test]
65686601
async fn spark_plan_dynamic_filter_preserves_metrics() {
65696602
use crate::execution::metrics::utils::to_native_metric_node;

‎spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2522,6 +2522,71 @@ class CometJoinSuite extends CometTestBase {
25222522
}
25232523
}
25242524

2525+
test("outer hash joins keep the order of a sorted bucketed table read by the native scan") {
2526+
// With spark.sql.legacy.bucketedTableScan.outputOrdering on, Spark reports a bucketed table's
2527+
// sort columns as the scan's order when every bucket has one file, and the sort above the
2528+
// join is dropped on that basis. The native scan reads each bucket's file in order, but it is
2529+
// a native leaf, not a JVM input, so the planner has to count it as a source whose order
2530+
// Spark reports, or the join moves its unmatched rows. A broadcast join, since a shuffled one
2531+
// would put an exchange between the scan and the join.
2532+
withTable("sorted_bucketed") {
2533+
// A single writer task writes one file per bucket.
2534+
(0 until 2000)
2535+
.map(i => (i % 40, i))
2536+
.toDF("k", "v")
2537+
.coalesce(1)
2538+
.write
2539+
.format("parquet")
2540+
.bucketBy(2, "k")
2541+
.sortBy("k")
2542+
.saveAsTable("sorted_bucketed")
2543+
withSQLConf(
2544+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
2545+
SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
2546+
SQLConf.BUCKETING_ENABLED.key -> "true",
2547+
SQLConf.AUTO_BUCKETED_SCAN_ENABLED.key -> "false",
2548+
SQLConf.LEGACY_BUCKETED_TABLE_SCAN_OUTPUT_ORDERING.key -> "true") {
2549+
withParquetTable((0 until 40 by 2).map(k => (k, k / 2)), "small") {
2550+
val streamed = spark.table("sorted_bucketed")
2551+
val build = spark.table("small")
2552+
val scanOrder = collectFirst(streamed.queryExecution.executedPlan) {
2553+
case scan: CometNativeScanExec if scan.bucketedScan => scan.outputOrdering
2554+
}
2555+
assert(scanOrder.exists(_.nonEmpty), streamed.queryExecution.executedPlan)
2556+
Seq("left_outer", "right_outer").foreach { joinType =>
2557+
val joined = if (joinType == "left_outer") {
2558+
streamed.join(build.hint("broadcast"), streamed("k") === build("_1"), joinType)
2559+
} else {
2560+
build.hint("broadcast").join(streamed, streamed("k") === build("_1"), joinType)
2561+
}
2562+
val df = joined
2563+
.select(streamed("k"), streamed("v"), build("_2").as("w"))
2564+
.sortWithinPartitions("k")
2565+
val plan = df.queryExecution.executedPlan
2566+
val context = s"$joinType\n$plan"
2567+
val joins = collect(plan) { case j: CometBroadcastHashJoinExec => j }
2568+
assert(joins.size == 1, context)
2569+
assert(
2570+
collect(joins.head) {
2571+
case s: CometNativeScanExec if s.bucketedScan => s
2572+
}.size == 1,
2573+
context)
2574+
assert(joins.head.nativeOp.getHashJoin.getStreamedSortOrdersCount > 0, context)
2575+
assert(
2576+
collect(plan) { case s @ (_: CometSortExec | _: SortExec) => s }.isEmpty,
2577+
context)
2578+
val expected = (0 until 2000).map { i =>
2579+
val k = i % 40
2580+
Row(k, i, if (k % 2 == 0) k / 2 else null)
2581+
}
2582+
QueryTest.sameRows(expected, df.collect().toSeq).foreach(fail(_))
2583+
assert(partitionsInOrder(df, keyOrder(Ascending)), context)
2584+
}
2585+
}
2586+
}
2587+
}
2588+
}
2589+
25252590
test("outer hash joins keep the streamed order for decimal, string and timestamp keys") {
25262591
// SortedInputExec passes the rows through, so the order is physically Spark's, but the serde
25272592
// and the native sort expression handle the key, and the sorted aggregate above the join finds

0 commit comments

Comments
 (0)