Skip to content

Commit a0e9887

Browse files
authored
fix: preserve empty projection when ser/de HashJoinExec and NestedLoopJoinExec (apache#23082)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> - Closes apache#23083 ## Rationale for this change `HashJoinExec` and `NestedLoopJoinExec` carry `projection: Option<Vec<usize>>` but the proto field is `repeated uint32`. Proto3 can't tell `None` from `Some(vec![])`, and the decoder treats both as `None`. `Some(vec![])` is reachable in real plans — `try_embed_projection` produces it for `SELECT count(1) … JOIN …` (apache#20191) — and after a round-trip the join silently switches from "emit zero columns" to "emit all columns". `FilterExec` has a workaround for the same limitation; these two execs were missed. ## What changes are included in this PR? Encode `Some(vec![])` as the single-element sentinel `[u32::MAX]` (never a valid column index); recognise it on decode. Everything else goes through unchanged. ```rust // encode projection: match exec.projection.as_ref() { None => Vec::new(), Some(v) if v.is_empty() => vec![u32::MAX], Some(v) => v.iter().map(|x| *x as u32).collect(), }, // decode let projection = match hashjoin.projection.as_slice() { [] => None, [u32::MAX] => Some(Vec::new()), indices => Some(indices.iter().map(|i| *i as usize).collect()), }; ``` Applied symmetrically to `HashJoinExec` and `NestedLoopJoinExec`. ## Are these changes tested? yes, add roundtrip test case ## Are there any user-facing changes? <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> <!-- If there are any breaking changes to public APIs, please add the `api change` label. -->
1 parent 3bee970 commit a0e9887

2 files changed

Lines changed: 84 additions & 25 deletions

File tree

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 31 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1498,16 +1498,15 @@ pub trait PhysicalPlanNodeExt: Sized {
14981498
protobuf::PartitionMode::Partitioned => PartitionMode::Partitioned,
14991499
protobuf::PartitionMode::Auto => PartitionMode::Auto,
15001500
};
1501-
let projection = if !hashjoin.projection.is_empty() {
1502-
Some(
1503-
hashjoin
1504-
.projection
1505-
.iter()
1506-
.map(|i| *i as usize)
1507-
.collect::<Vec<_>>(),
1508-
)
1509-
} else {
1510-
None
1501+
// Proto3 `repeated` cannot distinguish `None` from `Some(vec![])`. The latter
1502+
// is reachable via `try_embed_projection` for `SELECT count(1) … JOIN …` and
1503+
// changes the join's output schema, so the encoder reserves the single-element
1504+
// sentinel `[u32::MAX]` (never a valid column index) to mean "explicitly empty";
1505+
// every other state is sent as-is. See `try_from_hash_join_exec`.
1506+
let projection = match hashjoin.projection.as_slice() {
1507+
[] => None,
1508+
[u32::MAX] => Some(Vec::new()),
1509+
indices => Some(indices.iter().map(|i| *i as usize).collect()),
15111510
};
15121511
let mut hash_join = HashJoinExec::try_new(
15131512
left,
@@ -1925,15 +1924,13 @@ pub trait PhysicalPlanNodeExt: Sized {
19251924
})
19261925
.map_or(Ok(None), |v: Result<JoinFilter>| v.map(Some))?;
19271926

1928-
let projection = if !join.projection.is_empty() {
1929-
Some(
1930-
join.projection
1931-
.iter()
1932-
.map(|i| *i as usize)
1933-
.collect::<Vec<_>>(),
1934-
)
1935-
} else {
1936-
None
1927+
// See `try_into_hash_join_physical_plan` for the rationale behind the
1928+
// `[u32::MAX]` sentinel; `NestedLoopJoinExec` has the same `Option<Vec<usize>>`
1929+
// projection field and shares the proto3 `repeated` ambiguity.
1930+
let projection = match join.projection.as_slice() {
1931+
[] => None,
1932+
[u32::MAX] => Some(Vec::new()),
1933+
indices => Some(indices.iter().map(|i| *i as usize).collect()),
19371934
};
19381935

19391936
Ok(Arc::new(NestedLoopJoinExec::try_new(
@@ -2658,9 +2655,14 @@ pub trait PhysicalPlanNodeExt: Sized {
26582655
partition_mode: partition_mode.into(),
26592656
null_equality: null_equality.into(),
26602657
filter,
2661-
projection: exec.projection.as_ref().map_or_else(Vec::new, |v| {
2662-
v.iter().map(|x| *x as u32).collect::<Vec<u32>>()
2663-
}),
2658+
// Send `Some(vec![])` as `[u32::MAX]` (never a valid index) so the
2659+
// wire format can distinguish it from `None` (which stays empty).
2660+
// See `try_into_hash_join_physical_plan` for the matching decoder.
2661+
projection: match exec.projection.as_ref() {
2662+
None => Vec::new(),
2663+
Some(v) if v.is_empty() => vec![u32::MAX],
2664+
Some(v) => v.iter().map(|x| *x as u32).collect(),
2665+
},
26642666
null_aware: exec.null_aware,
26652667
dynamic_filter,
26662668
},
@@ -3449,9 +3451,13 @@ pub trait PhysicalPlanNodeExt: Sized {
34493451
right: Some(Box::new(right)),
34503452
join_type: join_type.into(),
34513453
filter,
3452-
projection: exec.projection().as_ref().map_or_else(Vec::new, |v| {
3453-
v.iter().map(|x| *x as u32).collect::<Vec<u32>>()
3454-
}),
3454+
// `[u32::MAX]` sentinel distinguishes `Some(vec![])` from `None`;
3455+
// see `try_from_hash_join_exec`.
3456+
projection: match exec.projection().as_ref() {
3457+
None => Vec::new(),
3458+
Some(v) if v.is_empty() => vec![u32::MAX],
3459+
Some(v) => v.iter().map(|x| *x as u32).collect(),
3460+
},
34553461
},
34563462
))),
34573463
})

‎datafusion/proto/tests/cases/roundtrip_physical_plan.rs‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -416,6 +416,59 @@ fn roundtrip_nested_loop_join() -> Result<()> {
416416
Ok(())
417417
}
418418

419+
/// Regression: proto3 `repeated` fields cannot distinguish "absent" from "empty",
420+
/// so a naive encoding collapses `Some(vec![])` and `None` into the same wire
421+
/// representation. `try_embed_projection` (DataFusion 53+) produces
422+
/// `HashJoinExec.projection = Some(vec![])` for `SELECT count(1) … JOIN …`,
423+
/// which previously round-tripped to `None` and caused downstream consumers (e.g.
424+
/// distributed Flight executors) to receive a different number of output
425+
/// columns than the planner declared. Verify all three states preserve.
426+
#[test]
427+
fn roundtrip_hash_join_projection_states() -> Result<()> {
428+
let field_a = Field::new("col", DataType::Int64, false);
429+
let schema_left = Arc::new(Schema::new(vec![field_a.clone()]));
430+
let schema_right = Arc::new(Schema::new(vec![field_a]));
431+
let on = vec![(
432+
Arc::new(Column::new("col", schema_left.index_of("col")?)) as _,
433+
Arc::new(Column::new("col", schema_right.index_of("col")?)) as _,
434+
)];
435+
436+
for projection in [None, Some(vec![]), Some(vec![0]), Some(vec![1])] {
437+
roundtrip_test(Arc::new(HashJoinExec::try_new(
438+
Arc::new(EmptyExec::new(schema_left.clone())),
439+
Arc::new(EmptyExec::new(schema_right.clone())),
440+
on.clone(),
441+
None,
442+
&JoinType::Inner,
443+
projection,
444+
PartitionMode::Partitioned,
445+
NullEquality::NullEqualsNothing,
446+
false,
447+
)?))?;
448+
}
449+
Ok(())
450+
}
451+
452+
/// Same regression coverage for `NestedLoopJoinExec`, which shares the
453+
/// `repeated uint32 projection` proto field shape with `HashJoinExec`.
454+
#[test]
455+
fn roundtrip_nested_loop_join_projection_states() -> Result<()> {
456+
let field_a = Field::new("col", DataType::Int64, false);
457+
let schema_left = Arc::new(Schema::new(vec![field_a.clone()]));
458+
let schema_right = Arc::new(Schema::new(vec![field_a]));
459+
460+
for projection in [None, Some(vec![]), Some(vec![0]), Some(vec![1])] {
461+
roundtrip_test(Arc::new(NestedLoopJoinExec::try_new(
462+
Arc::new(EmptyExec::new(schema_left.clone())),
463+
Arc::new(EmptyExec::new(schema_right.clone())),
464+
None,
465+
&JoinType::Inner,
466+
projection,
467+
)?))?;
468+
}
469+
Ok(())
470+
}
471+
419472
#[test]
420473
fn roundtrip_udwf() -> Result<()> {
421474
let field_a = Field::new("a", DataType::Int64, false);

0 commit comments

Comments
 (0)