Skip to content

Commit c499077

Browse files
haohuaijinthinkharderdev
authored andcommitted
fix: preserve empty projection when ser/de HashJoinExec and NestedLoopJoinExec (apache#23082)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> - Closes apache#23083 ## Rationale for this change `HashJoinExec` and `NestedLoopJoinExec` carry `projection: Option<Vec<usize>>` but the proto field is `repeated uint32`. Proto3 can't tell `None` from `Some(vec![])`, and the decoder treats both as `None`. `Some(vec![])` is reachable in real plans — `try_embed_projection` produces it for `SELECT count(1) … JOIN …` (apache#20191) — and after a round-trip the join silently switches from "emit zero columns" to "emit all columns". `FilterExec` has a workaround for the same limitation; these two execs were missed. ## What changes are included in this PR? Encode `Some(vec![])` as the single-element sentinel `[u32::MAX]` (never a valid column index); recognise it on decode. Everything else goes through unchanged. ```rust // encode projection: match exec.projection.as_ref() { None => Vec::new(), Some(v) if v.is_empty() => vec![u32::MAX], Some(v) => v.iter().map(|x| *x as u32).collect(), }, // decode let projection = match hashjoin.projection.as_slice() { [] => None, [u32::MAX] => Some(Vec::new()), indices => Some(indices.iter().map(|i| *i as usize).collect()), }; ``` Applied symmetrically to `HashJoinExec` and `NestedLoopJoinExec`. ## Are these changes tested? yes, add roundtrip test case ## Are there any user-facing changes? <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> <!-- If there are any breaking changes to public APIs, please add the `api change` label. -->
1 parent f26b4d3 commit c499077

2 files changed

Lines changed: 84 additions & 25 deletions

File tree

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 31 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1482,16 +1482,15 @@ impl protobuf::PhysicalPlanNode {
14821482
protobuf::PartitionMode::Partitioned => PartitionMode::Partitioned,
14831483
protobuf::PartitionMode::Auto => PartitionMode::Auto,
14841484
};
1485-
let projection = if !hashjoin.projection.is_empty() {
1486-
Some(
1487-
hashjoin
1488-
.projection
1489-
.iter()
1490-
.map(|i| *i as usize)
1491-
.collect::<Vec<_>>(),
1492-
)
1493-
} else {
1494-
None
1485+
// Proto3 `repeated` cannot distinguish `None` from `Some(vec![])`. The latter
1486+
// is reachable via `try_embed_projection` for `SELECT count(1) … JOIN …` and
1487+
// changes the join's output schema, so the encoder reserves the single-element
1488+
// sentinel `[u32::MAX]` (never a valid column index) to mean "explicitly empty";
1489+
// every other state is sent as-is. See `try_from_hash_join_exec`.
1490+
let projection = match hashjoin.projection.as_slice() {
1491+
[] => None,
1492+
[u32::MAX] => Some(Vec::new()),
1493+
indices => Some(indices.iter().map(|i| *i as usize).collect()),
14951494
};
14961495
let mut hash_join = HashJoinExec::try_new(
14971496
left,
@@ -1904,15 +1903,13 @@ impl protobuf::PhysicalPlanNode {
19041903
})
19051904
.map_or(Ok(None), |v: Result<JoinFilter>| v.map(Some))?;
19061905

1907-
let projection = if !join.projection.is_empty() {
1908-
Some(
1909-
join.projection
1910-
.iter()
1911-
.map(|i| *i as usize)
1912-
.collect::<Vec<_>>(),
1913-
)
1914-
} else {
1915-
None
1906+
// See `try_into_hash_join_physical_plan` for the rationale behind the
1907+
// `[u32::MAX]` sentinel; `NestedLoopJoinExec` has the same `Option<Vec<usize>>`
1908+
// projection field and shares the proto3 `repeated` ambiguity.
1909+
let projection = match join.projection.as_slice() {
1910+
[] => None,
1911+
[u32::MAX] => Some(Vec::new()),
1912+
indices => Some(indices.iter().map(|i| *i as usize).collect()),
19161913
};
19171914

19181915
Ok(Arc::new(NestedLoopJoinExec::try_new(
@@ -2602,9 +2599,14 @@ impl protobuf::PhysicalPlanNode {
26022599
partition_mode: partition_mode.into(),
26032600
null_equality: null_equality.into(),
26042601
filter,
2605-
projection: exec.projection.as_ref().map_or_else(Vec::new, |v| {
2606-
v.iter().map(|x| *x as u32).collect::<Vec<u32>>()
2607-
}),
2602+
// Send `Some(vec![])` as `[u32::MAX]` (never a valid index) so the
2603+
// wire format can distinguish it from `None` (which stays empty).
2604+
// See `try_into_hash_join_physical_plan` for the matching decoder.
2605+
projection: match exec.projection.as_ref() {
2606+
None => Vec::new(),
2607+
Some(v) if v.is_empty() => vec![u32::MAX],
2608+
Some(v) => v.iter().map(|x| *x as u32).collect(),
2609+
},
26082610
null_aware: exec.null_aware,
26092611
dynamic_filter,
26102612
},
@@ -3393,9 +3395,13 @@ impl protobuf::PhysicalPlanNode {
33933395
right: Some(Box::new(right)),
33943396
join_type: join_type.into(),
33953397
filter,
3396-
projection: exec.projection().as_ref().map_or_else(Vec::new, |v| {
3397-
v.iter().map(|x| *x as u32).collect::<Vec<u32>>()
3398-
}),
3398+
// `[u32::MAX]` sentinel distinguishes `Some(vec![])` from `None`;
3399+
// see `try_from_hash_join_exec`.
3400+
projection: match exec.projection().as_ref() {
3401+
None => Vec::new(),
3402+
Some(v) if v.is_empty() => vec![u32::MAX],
3403+
Some(v) => v.iter().map(|x| *x as u32).collect(),
3404+
},
33993405
},
34003406
))),
34013407
})

‎datafusion/proto/tests/cases/roundtrip_physical_plan.rs‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,59 @@ fn roundtrip_nested_loop_join() -> Result<()> {
347347
Ok(())
348348
}
349349

350+
/// Regression: proto3 `repeated` fields cannot distinguish "absent" from "empty",
351+
/// so a naive encoding collapses `Some(vec![])` and `None` into the same wire
352+
/// representation. `try_embed_projection` (DataFusion 53+) produces
353+
/// `HashJoinExec.projection = Some(vec![])` for `SELECT count(1) … JOIN …`,
354+
/// which previously round-tripped to `None` and caused downstream consumers (e.g.
355+
/// distributed Flight executors) to receive a different number of output
356+
/// columns than the planner declared. Verify all three states preserve.
357+
#[test]
358+
fn roundtrip_hash_join_projection_states() -> Result<()> {
359+
let field_a = Field::new("col", DataType::Int64, false);
360+
let schema_left = Arc::new(Schema::new(vec![field_a.clone()]));
361+
let schema_right = Arc::new(Schema::new(vec![field_a]));
362+
let on = vec![(
363+
Arc::new(Column::new("col", schema_left.index_of("col")?)) as _,
364+
Arc::new(Column::new("col", schema_right.index_of("col")?)) as _,
365+
)];
366+
367+
for projection in [None, Some(vec![]), Some(vec![0]), Some(vec![1])] {
368+
roundtrip_test(Arc::new(HashJoinExec::try_new(
369+
Arc::new(EmptyExec::new(schema_left.clone())),
370+
Arc::new(EmptyExec::new(schema_right.clone())),
371+
on.clone(),
372+
None,
373+
&JoinType::Inner,
374+
projection,
375+
PartitionMode::Partitioned,
376+
NullEquality::NullEqualsNothing,
377+
false,
378+
)?))?;
379+
}
380+
Ok(())
381+
}
382+
383+
/// Same regression coverage for `NestedLoopJoinExec`, which shares the
384+
/// `repeated uint32 projection` proto field shape with `HashJoinExec`.
385+
#[test]
386+
fn roundtrip_nested_loop_join_projection_states() -> Result<()> {
387+
let field_a = Field::new("col", DataType::Int64, false);
388+
let schema_left = Arc::new(Schema::new(vec![field_a.clone()]));
389+
let schema_right = Arc::new(Schema::new(vec![field_a]));
390+
391+
for projection in [None, Some(vec![]), Some(vec![0]), Some(vec![1])] {
392+
roundtrip_test(Arc::new(NestedLoopJoinExec::try_new(
393+
Arc::new(EmptyExec::new(schema_left.clone())),
394+
Arc::new(EmptyExec::new(schema_right.clone())),
395+
None,
396+
&JoinType::Inner,
397+
projection,
398+
)?))?;
399+
}
400+
Ok(())
401+
}
402+
350403
#[test]
351404
fn roundtrip_udwf() -> Result<()> {
352405
let field_a = Field::new("a", DataType::Int64, false);

0 commit comments

Comments
 (0)