Skip to content

Commit 16d1605

Browse files
committed
fix: InterleaveExec fallback to UnionExec when children partitioning diverges
When optimizer rewrites (e.g. join_selection changing join modes, or additional EnforceDistribution passes) modify InterleaveExec children's plan structure, their output partitioning may no longer be consistent. Previously this caused an assertion panic. Now it gracefully falls back to UnionExec — correctness is preserved, only the interleave optimization is lost. Closes apache#21826
1 parent 14c18ec commit 16d1605

1 file changed

Lines changed: 98 additions & 7 deletions

File tree

‎datafusion/physical-plan/src/union.rs‎

Lines changed: 98 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -612,13 +612,21 @@ impl ExecutionPlan for InterleaveExec {
612612
self: Arc<Self>,
613613
children: Vec<Arc<dyn ExecutionPlan>>,
614614
) -> Result<Arc<dyn ExecutionPlan>> {
615-
// New children are no longer interleavable, which might be a bug of optimization rewrite.
616-
assert_or_internal_err!(
617-
can_interleave(children.iter()),
618-
"Can not create InterleaveExec: new children can not be interleaved"
619-
);
620-
check_if_same_properties!(self, children);
621-
Ok(Arc::new(InterleaveExec::try_new(children)?))
615+
// After optimizer rewrites (e.g. join_selection changing join modes,
616+
// or additional EnforceDistribution passes), children may no longer
617+
// have consistent hash partitioning. Fall back to UnionExec instead
618+
// of panicking — correctness is preserved, only the interleave
619+
// optimization is lost.
620+
if can_interleave(children.iter()) {
621+
check_if_same_properties!(self, children);
622+
Ok(Arc::new(InterleaveExec::try_new(children)?))
623+
} else {
624+
warn!(
625+
"InterleaveExec children no longer interleavable after optimizer rewrite, \
626+
falling back to UnionExec"
627+
);
628+
UnionExec::try_new(children)
629+
}
622630
}
623631

624632
fn execute(
@@ -1452,4 +1460,87 @@ mod tests {
14521460
));
14531461
Ok(())
14541462
}
1463+
1464+
use crate::repartition::RepartitionExec;
1465+
1466+
/// Test that InterleaveExec::with_new_children falls back to UnionExec
1467+
/// when children's partitioning diverges after optimizer rewrites.
1468+
#[tokio::test]
1469+
async fn test_interleave_fallback_to_union_on_partitioning_mismatch() -> Result<()> {
1470+
let schema = create_test_schema()?;
1471+
let memory_exec1 = TestMemoryExec::try_new_exec(&[], Arc::clone(&schema), None)?;
1472+
let memory_exec2 = TestMemoryExec::try_new_exec(&[], Arc::clone(&schema), None)?;
1473+
1474+
// Wrap in RepartitionExec to get Hash partitioning
1475+
let hash_child1 = Arc::new(RepartitionExec::try_new(
1476+
memory_exec1,
1477+
Partitioning::Hash(vec![col("a", &schema)?], 8),
1478+
)?);
1479+
let hash_child2 = Arc::new(RepartitionExec::try_new(
1480+
memory_exec2,
1481+
Partitioning::Hash(vec![col("a", &schema)?], 8),
1482+
)?);
1483+
1484+
// Create InterleaveExec — should succeed
1485+
let interleave = Arc::new(InterleaveExec::try_new(vec![
1486+
Arc::clone(&hash_child1) as _,
1487+
Arc::clone(&hash_child2) as _,
1488+
])?);
1489+
assert_eq!(interleave.name(), "InterleaveExec");
1490+
1491+
// Simulate optimizer rewrite: one child changed to RoundRobin
1492+
let memory_exec3 = TestMemoryExec::try_new_exec(&[], Arc::clone(&schema), None)?;
1493+
let mismatched_child = Arc::new(RepartitionExec::try_new(
1494+
memory_exec3,
1495+
Partitioning::RoundRobinBatch(8),
1496+
)?);
1497+
1498+
// with_new_children should NOT panic — falls back to UnionExec
1499+
let result = interleave
1500+
.with_new_children(vec![hash_child1 as _, mismatched_child as _])?;
1501+
1502+
assert_eq!(result.name(), "UnionExec");
1503+
Ok(())
1504+
}
1505+
1506+
/// Test that InterleaveExec::with_new_children preserves InterleaveExec
1507+
/// when children's partitioning remains consistent.
1508+
#[tokio::test]
1509+
async fn test_interleave_preserved_when_partitioning_matches() -> Result<()> {
1510+
let schema = create_test_schema()?;
1511+
let memory_exec1 = TestMemoryExec::try_new_exec(&[], Arc::clone(&schema), None)?;
1512+
let memory_exec2 = TestMemoryExec::try_new_exec(&[], Arc::clone(&schema), None)?;
1513+
1514+
let hash_child1 = Arc::new(RepartitionExec::try_new(
1515+
memory_exec1,
1516+
Partitioning::Hash(vec![col("a", &schema)?], 8),
1517+
)?);
1518+
let hash_child2 = Arc::new(RepartitionExec::try_new(
1519+
memory_exec2,
1520+
Partitioning::Hash(vec![col("a", &schema)?], 8),
1521+
)?);
1522+
1523+
let interleave = Arc::new(InterleaveExec::try_new(vec![
1524+
Arc::clone(&hash_child1) as _,
1525+
Arc::clone(&hash_child2) as _,
1526+
])?);
1527+
1528+
// New children with same Hash partitioning
1529+
let memory_exec3 = TestMemoryExec::try_new_exec(&[], Arc::clone(&schema), None)?;
1530+
let memory_exec4 = TestMemoryExec::try_new_exec(&[], Arc::clone(&schema), None)?;
1531+
let new_child1 = Arc::new(RepartitionExec::try_new(
1532+
memory_exec3,
1533+
Partitioning::Hash(vec![col("a", &schema)?], 8),
1534+
)?);
1535+
let new_child2 = Arc::new(RepartitionExec::try_new(
1536+
memory_exec4,
1537+
Partitioning::Hash(vec![col("a", &schema)?], 8),
1538+
)?);
1539+
1540+
let result =
1541+
interleave.with_new_children(vec![new_child1 as _, new_child2 as _])?;
1542+
1543+
assert_eq!(result.name(), "InterleaveExec");
1544+
Ok(())
1545+
}
14551546
}

0 commit comments

Comments
 (0)