Skip to content

Commit 9c5dcd0

Browse files
committed
refactor: send only a window's required order and merge overlapping tests
with_declared_order extends a partial order at the upper window, so the serde no longer sends the keys a window's child reports past the ones the window requires. The two stacked-window tests become one over a single cache, the EmptyExec unit test folds into the extension test, and redundant assertions and comments are trimmed.
1 parent dc94ccf commit 9c5dcd0

7 files changed

Lines changed: 98 additions & 211 deletions

File tree

‎docs/source/contributor-guide/debugging.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -260,8 +260,8 @@ so `spark.comet.debug.enabled` and `COMET_LOG_LEVEL` do not affect it.
260260
#### Example: dumping window operator output
261261

262262
In `WindowBuilder::build` in `native/core/src/execution/planner/window.rs`,
263-
the window operator builds `window_agg` over `input`, the child plan with any
264-
order Spark sent declared on it, either a `BoundedWindowAggExec` or a
263+
the window operator builds `window_agg` over `input`, the child plan with
264+
Spark's input order declared, either a `BoundedWindowAggExec` or a
265265
`WindowAggExec` depending on whether every window expression runs with bounded
266266
memory. It then optionally wraps `window_agg` in a casting `ProjectionExec` and
267267
builds the `SparkPlan` from either the projection or `window_agg`. To see the raw

‎native/core/src/execution/planner.rs‎

Lines changed: 6 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -4270,35 +4270,10 @@ mod tests {
42704270
assert!(!PhysicalPlanner::keeps_source_order(&scan(2)));
42714271
}
42724272

4273-
#[test]
4274-
fn declared_order_reaches_an_empty_bucket() {
4275-
// A bucket with no file is planned as an `EmptyExec`. A window over it checks its input's
4276-
// order before polling it, so the order Spark reports has to be declared there as well.
4277-
let schema = Arc::new(Schema::new(vec![Field::new(
4278-
"col_0",
4279-
DataType::Int32,
4280-
true,
4281-
)]));
4282-
let empty: Arc<dyn ExecutionPlan> =
4283-
Arc::new(datafusion::physical_plan::empty::EmptyExec::new(schema));
4284-
let input = PhysicalPlanner::default()
4285-
.with_declared_order(empty, &[create_sort_order(0, 3, false, true)])
4286-
.unwrap();
4287-
assert_eq!("SortedInputExec", input.name());
4288-
let partition_by: Vec<Arc<dyn PhysicalExpr>> = vec![Arc::new(Column::new("col_0", 0))];
4289-
assert_eq!(
4290-
1,
4291-
get_ordered_partition_by_indices(&partition_by, &input)
4292-
.unwrap()
4293-
.len()
4294-
);
4295-
}
4296-
42974273
#[test]
42984274
fn declared_order_extends_an_order_the_input_declares() {
4299-
// Two aliases of one column share an order, so a window over the window for one alias can
4300-
// need the other's order while its input declares only the first. That order is declared
4301-
// next to the input's, unless an operator in between may have moved rows.
4275+
// An order is declared over an empty bucket, and next to the order the input declares,
4276+
// unless an operator in between may have moved rows.
43024277
let schema = Arc::new(Schema::new(vec![
43034278
Field::new("col_0", DataType::Int32, true),
43044279
Field::new("col_1", DataType::Int32, true),
@@ -4315,6 +4290,7 @@ mod tests {
43154290
let empty: Arc<dyn ExecutionPlan> =
43164291
Arc::new(datafusion::physical_plan::empty::EmptyExec::new(schema));
43174292
let by_col_0 = declare(&empty, 0);
4293+
assert_eq!("SortedInputExec", by_col_0.name());
43184294
assert!(Arc::ptr_eq(&declare(&by_col_0, 0), &by_col_0));
43194295

43204296
let both = declare(&by_col_0, 1);
@@ -4525,8 +4501,8 @@ mod tests {
45254501

45264502
#[test]
45274503
fn window_declares_input_order_on_a_scan() {
4528-
// Both window operators need their partition keys' order declared, and over a `ScanExec`
4529-
// only the order Spark sent declares it. `percent_rank` does not run with bounded memory.
4504+
// Both window operators need their partition keys' order, which a `ScanExec` declares
4505+
// only through the order Spark sent. `percent_rank` runs in `WindowAggExec`.
45304506
let partition_by: Vec<Arc<dyn PhysicalExpr>> = vec![Arc::new(Column::new("col_0", 0))];
45314507
for (func, operator) in [
45324508
("row_number", "BoundedWindowAggExec"),
@@ -4544,14 +4520,7 @@ mod tests {
45444520
Arc::clone(planned.native_plan.children()[0])
45454521
};
45464522
let input = window_input(vec![create_sort_order(0, 3, false, true)]);
4547-
let sorted = input
4548-
.downcast_ref::<crate::execution::operators::SortedInputExec>()
4549-
.expect("the window's input declares the order Spark sent");
4550-
assert_eq!("ScanExec", sorted.children()[0].name());
4551-
assert_eq!(
4552-
"col_0@0 ASC",
4553-
input.properties().output_ordering().unwrap().to_string()
4554-
);
4523+
assert_eq!("SortedInputExec", input.name());
45554524
let ordered = get_ordered_partition_by_indices(&partition_by, &input).unwrap();
45564525
assert_eq!(1, ordered.len(), "{func}");
45574526

‎native/core/src/execution/planner/join.rs‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -409,10 +409,8 @@ impl PhysicalPlanner {
409409
/// scan the rows arrive through declares none. Only that order is declared: when an operator
410410
/// in between may have moved rows, `plan` is left as it is.
411411
///
412-
/// An input can declare part of the order already, as a window does over an input declared
413-
/// for the window below it. Spark may know more than that declaration: two aliases of one
414-
/// column share an order, so the window above may need the other alias's order. The rest is
415-
/// declared next to what `plan` declares.
412+
/// An input that declares only part of `sort_orders`, such as a window over another window,
413+
/// gets them declared next to its own order.
416414
///
417415
/// The serde sends an order only when the Comet operators of a join's streamed side or of a
418416
/// window's input report it, and it checks that nothing under them moves rows
@@ -429,7 +427,7 @@ impl PhysicalPlanner {
429427
plan: Arc<dyn ExecutionPlan>,
430428
sort_orders: &[Expr],
431429
) -> Result<Arc<dyn ExecutionPlan>, ExecutionError> {
432-
if !Self::keeps_source_order(&plan) {
430+
if sort_orders.is_empty() || !Self::keeps_source_order(&plan) {
433431
return Ok(plan);
434432
}
435433
let exprs = sort_orders
@@ -452,10 +450,10 @@ impl PhysicalPlanner {
452450
/// keep their input's order, so that the order Spark reports for them still holds. A source is
453451
/// an input from the JVM, or a scan of files, which reads the files of its one file group one
454452
/// after the other: Spark reports an order for a file scan only when every partition reads one
455-
/// sorted file, as for a sorted bucketed table with one file per bucket. A bucket with no file
456-
/// is planned as an `EmptyExec`, which has no rows to be out of order. Any other operator may
457-
/// have moved rows, such as the null-aware anti join, whose streamed side Comet builds on, and
458-
/// declaring Spark's order over it would be wrong.
453+
/// sorted file, as for a sorted bucketed table with one file per bucket. An `EmptyExec`, a
454+
/// bucket with no file, has no rows to be out of order. Any other operator may have moved
455+
/// rows, such as the null-aware anti join, whose streamed side Comet builds on, and declaring
456+
/// Spark's order over it would be wrong.
459457
pub(super) fn keeps_source_order(plan: &Arc<dyn ExecutionPlan>) -> bool {
460458
if plan.is::<ScanExec>() || plan.is::<ShuffleScanExec>() || plan.is::<EmptyExec>() {
461459
return true;

‎native/proto/src/proto/operator.proto‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1233,9 +1233,8 @@ message Window {
12331233
repeated spark.spark_expression.Expr order_by_list = 2;
12341234
repeated spark.spark_expression.Expr partition_by_list = 3;
12351235
Operator child = 4;
1236-
// The partition keys ascending, then the order keys, then any further keys the input reports,
1237-
// when Spark planned no sort below the window because its input reports that order. Empty
1238-
// otherwise.
1236+
// The partition keys ascending, then the order keys, when Spark planned no sort below the
1237+
// window because its input reports that order. Empty otherwise.
12391238
repeated spark.spark_expression.Expr input_sort_orders = 5;
12401239
}
12411240

‎spark/src/main/scala/org/apache/spark/sql/comet/CometWindowExec.scala‎

Lines changed: 7 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ import org.apache.comet.{CometConf, ConfigEntry}
3636
import org.apache.comet.CometSparkSessionExtensions.withFallbackReason
3737
import org.apache.comet.serde.{AggSerde, CometOperatorSerde, CometSortOrder, LiteralOuterClass, OperatorOuterClass}
3838
import org.apache.comet.serde.OperatorOuterClass.Operator
39-
import org.apache.comet.serde.QueryPlanSerde.{aggExprToProto, exprToProto, exprToProtoWithFallbackReasons, hasNonDefaultStringCollation, liftFallbackReasons, scalarFunctionExprToProto, serializeDataType}
39+
import org.apache.comet.serde.QueryPlanSerde.{aggExprToProto, exprToProto, hasNonDefaultStringCollation, liftFallbackReasons, scalarFunctionExprToProto, serializeDataType}
4040

4141
object CometWindowExec extends CometOperatorSerde[WindowExec] {
4242

@@ -81,12 +81,11 @@ object CometWindowExec extends CometOperatorSerde[WindowExec] {
8181

8282
if (windowExprProto.forall(_.isDefined) && partitionExprs.forall(_.isDefined)
8383
&& sortOrders.forall(_.isDefined)) {
84-
// Spark plans no sort below the window when its input reports the order the window needs,
85-
// a sorted cache for instance, but rows from the JVM arrive through a ScanExec that
86-
// declares none, and the native window rejects them. Send the order for the planner to
87-
// declare (see SortedInputExec), unless a native sort below declares it or the reported
88-
// order may be false. A key Comet cannot express or a collated string key, which the
89-
// native side would order by raw bytes, sends the window back to Spark.
84+
// Spark plans no sort below the window when its input already reports the order the window
85+
// needs, but rows from the JVM arrive through a ScanExec that declares none. Send that order
86+
// for the planner to declare (see SortedInputExec), unless a native sort below declares it
87+
// or it may be false. A key Comet cannot express, or a collated string key, sends the window
88+
// back to Spark.
9089
val requiredOrdering = op.requiredChildOrdering.head
9190
val inputSortOrders =
9291
if (requiredOrdering.nonEmpty && !op.child.isInstanceOf[CometSortExec] &&
@@ -103,25 +102,7 @@ object CometWindowExec extends CometOperatorSerde[WindowExec] {
103102
"natively")
104103
return None
105104
}
106-
// The keys the child reports past the required ones hold too, and this window reports
107-
// them, so a window above in the same native plan may rely on them. Send them up to the
108-
// first one that cannot be expressed, in a scope of their own, so that a key left out
109-
// records no fallback reason. Only a key's child is sent, and its same-order
110-
// expressions may name columns the child no longer outputs.
111-
val rest = op.child.outputOrdering
112-
.drop(requiredOrdering.length)
113-
.iterator
114-
.map { order =>
115-
if (hasNonDefaultStringCollation(order.dataType) ||
116-
!order.child.references.subsetOf(op.child.outputSet)) {
117-
None
118-
} else {
119-
exprToProtoWithFallbackReasons(order, op.child.output)._1
120-
}
121-
}
122-
.takeWhile(_.isDefined)
123-
.flatten
124-
orders.flatten ++ rest
105+
orders.flatten
125106
} else {
126107
Nil
127108
}

0 commit comments

Comments
 (0)