Skip to content

Commit 94aaa19

Browse files
committed
test: cover WindowAggExec, nulls, descending keys and the collated fallback
From review: - The cache test also runs a frame up to UNBOUNDED FOLLOWING, which runs in WindowAggExec, over a nullable partition key and a descending order key, so the declared order has to match the cache's. - The planner test covers both window operators through the partition order lookup they share, since WindowAggExec fails only when its stream starts, not in partition_by_sort_keys. - A Spark 4.x test checks that a window over a cache sorted by a collated key stays in Spark for the new reason and returns Spark's rows. - The debugging guide's window snippet builds the operators over `input`.
1 parent 8bc8908 commit 94aaa19

3 files changed

Lines changed: 109 additions & 56 deletions

File tree

‎docs/source/contributor-guide/debugging.md‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -260,7 +260,8 @@ so `spark.comet.debug.enabled` and `COMET_LOG_LEVEL` do not affect it.
260260
#### Example: dumping window operator output
261261

262262
In `WindowBuilder::build` in `native/core/src/execution/planner/window.rs`,
263-
the window operator builds `window_agg`, either a `BoundedWindowAggExec` or a
263+
the window operator builds `window_agg` over `input`, the child plan with any
264+
order Spark sent declared on it, either a `BoundedWindowAggExec` or a
264265
`WindowAggExec` depending on whether every window expression runs with bounded
265266
memory. It then optionally wraps `window_agg` in a casting `ProjectionExec` and
266267
builds the `SparkPlan` from either the projection or `window_agg`. To see the raw
@@ -271,14 +272,14 @@ projection:
271272
let window_agg: Arc<dyn ExecutionPlan> = if all_bounded {
272273
Arc::new(BoundedWindowAggExec::try_new(
273274
window_expr,
274-
Arc::clone(&child.native_plan),
275+
input,
275276
InputOrderMode::Sorted,
276277
!partition_exprs.is_empty(),
277278
)?)
278279
} else {
279280
Arc::new(WindowAggExec::try_new(
280281
window_expr,
281-
Arc::clone(&child.native_plan),
282+
input,
282283
!partition_exprs.is_empty(),
283284
)?)
284285
};

‎native/core/src/execution/planner.rs‎

Lines changed: 44 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -4337,9 +4337,10 @@ mod tests {
43374337
}
43384338
}
43394339

4340-
/// `row_number() OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)`, returning
4340+
/// `func() OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)`, returning
43414341
/// `result_type`.
4342-
fn row_number_window_expr(
4342+
fn window_function_expr(
4343+
func: &str,
43434344
result_type: Option<spark_expression::DataType>,
43444345
) -> spark_operator::WindowExpr {
43454346
use spark_operator::{
@@ -4351,7 +4352,7 @@ mod tests {
43514352
WindowExpr {
43524353
built_in_window_function: Some(Expr {
43534354
expr_struct: Some(ScalarFunc(spark_expression::ScalarFunc {
4354-
func: "row_number".to_string(),
4355+
func: func.to_string(),
43554356
..Default::default()
43564357
})),
43574358
query_context: None,
@@ -4386,7 +4387,7 @@ mod tests {
43864387
let planner = PhysicalPlanner::new(Arc::clone(&session), 0);
43874388
// `row_number` returns UInt64 natively and Spark expects Int32, so the planner adds a
43884389
// casting projection above the window.
4389-
let row_number = row_number_window_expr(Some(create_proto_datatype()));
4390+
let row_number = window_function_expr("row_number", Some(create_proto_datatype()));
43904391
let operator = Operator {
43914392
children: vec![create_scan()],
43924393
op_struct: Some(OpStruct::Window(Box::new(spark_operator::Window {
@@ -4441,12 +4442,12 @@ mod tests {
44414442
);
44424443
}
44434444

4444-
/// A window partitioned and ordered by `col_0` over `input`.
4445-
fn partitioned_window(input: Operator, input_sort_orders: Vec<Expr>) -> Operator {
4445+
/// `func` over a window partitioned and ordered by `col_0` over `input`.
4446+
fn partitioned_window(func: &str, input: Operator, input_sort_orders: Vec<Expr>) -> Operator {
44464447
Operator {
44474448
children: vec![input],
44484449
op_struct: Some(OpStruct::Window(Box::new(spark_operator::Window {
4449-
window_expr: vec![row_number_window_expr(None)],
4450+
window_expr: vec![window_function_expr(func, None)],
44504451
partition_by_list: vec![create_bound_reference(0)],
44514452
order_by_list: vec![create_sort_order(0, 3, false, true)],
44524453
input_sort_orders,
@@ -4458,39 +4459,42 @@ mod tests {
44584459

44594460
#[test]
44604461
fn window_declares_input_order_on_a_scan() {
4461-
// The window finds its partition keys' order over a `ScanExec` only once the planner
4462-
// declares the order Spark sent.
4463-
let window_over = |input_sort_orders| {
4464-
let (_, _, planned) = PhysicalPlanner::default()
4465-
.create_plan(
4466-
&partitioned_window(create_scan(), input_sort_orders),
4467-
&mut vec![],
4468-
1,
4469-
)
4470-
.unwrap();
4471-
let window = planned
4472-
.native_plan
4473-
.downcast_ref::<datafusion::physical_plan::windows::BoundedWindowAggExec>()
4474-
.expect("a bounded window");
4475-
(Arc::clone(window.input()), window.partition_by_sort_keys())
4476-
};
4477-
let (input, keys) = window_over(vec![create_sort_order(0, 3, false, true)]);
4478-
let sorted = input
4479-
.downcast_ref::<crate::execution::operators::SortedInputExec>()
4480-
.expect("the window's input declares the order Spark sent");
4481-
assert_eq!("ScanExec", sorted.children()[0].name());
4482-
assert_eq!(
4483-
"col_0@0 ASC",
4484-
input.properties().output_ordering().unwrap().to_string()
4485-
);
4486-
assert_eq!(1, keys.unwrap().len());
4487-
4488-
let (input, keys) = window_over(vec![]);
4489-
assert_eq!("ScanExec", input.name());
4490-
assert!(keys
4491-
.unwrap_err()
4492-
.to_string()
4493-
.contains("Expects PARTITION BY expression to be ordered"));
4462+
// Both window operators need their partition keys' order declared, and over a `ScanExec`
4463+
// only the order Spark sent declares it. `percent_rank` does not run with bounded memory.
4464+
use datafusion::physical_plan::windows::get_ordered_partition_by_indices;
4465+
let partition_by: Vec<Arc<dyn PhysicalExpr>> = vec![Arc::new(Column::new("col_0", 0))];
4466+
for (func, operator) in [
4467+
("row_number", "BoundedWindowAggExec"),
4468+
("percent_rank", "WindowAggExec"),
4469+
] {
4470+
let window_input = |input_sort_orders| {
4471+
let (_, _, planned) = PhysicalPlanner::default()
4472+
.create_plan(
4473+
&partitioned_window(func, create_scan(), input_sort_orders),
4474+
&mut vec![],
4475+
1,
4476+
)
4477+
.unwrap();
4478+
assert_eq!(operator, planned.native_plan.name());
4479+
Arc::clone(planned.native_plan.children()[0])
4480+
};
4481+
let input = window_input(vec![create_sort_order(0, 3, false, true)]);
4482+
let sorted = input
4483+
.downcast_ref::<crate::execution::operators::SortedInputExec>()
4484+
.expect("the window's input declares the order Spark sent");
4485+
assert_eq!("ScanExec", sorted.children()[0].name());
4486+
assert_eq!(
4487+
"col_0@0 ASC",
4488+
input.properties().output_ordering().unwrap().to_string()
4489+
);
4490+
let ordered = get_ordered_partition_by_indices(&partition_by, &input).unwrap();
4491+
assert_eq!(1, ordered.len(), "{func}");
4492+
4493+
let input = window_input(vec![]);
4494+
assert_eq!("ScanExec", input.name());
4495+
let ordered = get_ordered_partition_by_indices(&partition_by, &input).unwrap();
4496+
assert!(ordered.is_empty(), "{func}");
4497+
}
44944498
}
44954499

44964500
#[test]

‎spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala‎

Lines changed: 61 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3695,7 +3695,19 @@ class CometInMemoryCacheSuite extends CometTestBase {
36953695
test("a native window over a sorted cache declares the cache's order") {
36963696
// Spark plans no sort below a window over the cache's sort keys, and the native window
36973697
// rejected the scan, which declares no order, with "Expects PARTITION BY expression to be
3698-
// ordered" (#7223).
3698+
// ordered" (#7223). The partition key is nullable and the order key descending, so the
3699+
// declared order has to match the cache's direction and null placement. dense_rank runs in
3700+
// DataFusion's BoundedWindowAggExec, and the frame up to UNBOUNDED FOLLOWING in WindowAggExec.
3701+
// Within a key, v = k + 20 * j for j in 0 until 100, and a NULL key holds the multiples of 20.
3702+
val key = (v: Long) => if (v % 20 == 0) null else v % 20
3703+
val queries = Seq(
3704+
"dense_rank() OVER (PARTITION BY k ORDER BY v DESC)" ->
3705+
((v: Long) => (100 - v / 20).toInt),
3706+
"sum(v) OVER (PARTITION BY k ORDER BY v DESC " +
3707+
"ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)" -> { (v: Long) =>
3708+
val j = v / 20
3709+
(j + 1) * (v % 20) + 10 * j * (j + 1)
3710+
})
36993711
Seq(true, false).foreach { nativeCache =>
37003712
withConversions(
37013713
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
@@ -3705,22 +3717,23 @@ class CometInMemoryCacheSuite extends CometTestBase {
37053717
try {
37063718
val cached = spark
37073719
.range(2000)
3708-
.selectExpr("id % 20 AS k", "id AS v")
3720+
.selectExpr("CASE WHEN id % 20 = 0 THEN NULL ELSE id % 20 END AS k", "id AS v")
37093721
.repartition(2, col("k"))
3710-
.sortWithinPartitions("k", "v")
3722+
.sortWithinPartitions(col("k"), col("v").desc)
37113723
.cache()
37123724
cached.count()
37133725

3714-
val df =
3715-
cached.selectExpr("k", "v", "dense_rank() OVER (PARTITION BY k ORDER BY v) AS r")
3716-
val expected = (0L until 2000L).map(v => Row(v % 20, v, (v / 20 + 1).toInt))
3717-
QueryTest.sameRows(expected, df.collect().toSeq).foreach(fail(_))
3718-
val plan = df.queryExecution.executedPlan
3719-
withClue(s"nativeCache=$nativeCache\n$plan\n") {
3720-
val windows = collect(plan) { case w: CometWindowExec => w }
3721-
assert(windows.size == 1)
3722-
assert(windows.head.nativeOp.getWindow.getInputSortOrdersCount == 2)
3723-
assert(collect(plan) { case s @ (_: CometSortExec | _: SortExec) => s }.isEmpty)
3726+
queries.foreach { case (window, expected) =>
3727+
val df = cached.selectExpr("k", "v", s"$window AS w")
3728+
val rows = (0L until 2000L).map(v => Row(key(v), v, expected(v)))
3729+
QueryTest.sameRows(rows, df.collect().toSeq).foreach(fail(_))
3730+
val plan = df.queryExecution.executedPlan
3731+
withClue(s"nativeCache=$nativeCache, $window\n$plan\n") {
3732+
val windows = collect(plan) { case w: CometWindowExec => w }
3733+
assert(windows.size == 1)
3734+
assert(windows.head.nativeOp.getWindow.getInputSortOrdersCount == 2)
3735+
assert(collect(plan) { case s @ (_: CometSortExec | _: SortExec) => s }.isEmpty)
3736+
}
37243737
}
37253738
} finally {
37263739
spark.catalog.clearCache()
@@ -3729,6 +3742,41 @@ class CometInMemoryCacheSuite extends CometTestBase {
37293742
}
37303743
}
37313744

3745+
test("a window over a cache sorted by a collated string key stays in Spark") {
3746+
assume(isSpark40Plus, "collated string types require Spark 4.0+")
3747+
// The native side would declare the order of a collated key in raw byte order, which is not
3748+
// the order Spark sorted the cache in, so the window falls back instead.
3749+
withConversions(
3750+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
3751+
SQLConf.SHUFFLE_PARTITIONS.key -> "2") {
3752+
try {
3753+
val cached = spark
3754+
.range(200)
3755+
.selectExpr(
3756+
"CAST(id % 4 AS STRING) || CASE WHEN id % 8 < 4 THEN 'a' ELSE 'A' END " +
3757+
"COLLATE UTF8_LCASE AS k",
3758+
"id AS v")
3759+
.repartition(2, col("k"))
3760+
.sortWithinPartitions("k", "v")
3761+
.cache()
3762+
cached.count()
3763+
3764+
val df = cached.selectExpr("k", "v", "dense_rank() OVER (PARTITION BY k ORDER BY v) AS r")
3765+
// UTF8_LCASE puts 'a' and 'A' in one partition, so each of the four keys holds 50 rows.
3766+
val rows = (0L until 200L).map { v =>
3767+
Row(s"${v % 4}${if (v % 8 < 4) "a" else "A"}", v, (v / 4 + 1).toInt)
3768+
}
3769+
QueryTest.sameRows(rows, df.collect().toSeq).foreach(fail(_))
3770+
val plan = df.queryExecution.executedPlan
3771+
assert(collect(plan) { case w: CometWindowExec => w }.isEmpty, plan)
3772+
val reasons = new ExtendedExplainInfo().getFallbackReasons(plan)
3773+
assert(reasons.exists(_.contains("the order of the window's input")), reasons)
3774+
} finally {
3775+
spark.catalog.clearCache()
3776+
}
3777+
}
3778+
}
3779+
37323780
test("an aggregate on the key above a hash join over a sorted cache groups each key once") {
37333781
// The order the join declares on its probe input reaches past the join, so the aggregate
37343782
// above it in the same native plan groups in sorted, streaming mode. That is right while

0 commit comments

Comments
 (0)