Skip to content

Commit ea7ae7e

Browse files
haohuaijinDandandan
authored andcommitted
fix: preserve empty projection when ser/de HashJoinExec and NestedLoopJoinExec (apache#23082)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> - Closes apache#23083 ## Rationale for this change `HashJoinExec` and `NestedLoopJoinExec` carry `projection: Option<Vec<usize>>` but the proto field is `repeated uint32`. Proto3 can't tell `None` from `Some(vec![])`, and the decoder treats both as `None`. `Some(vec![])` is reachable in real plans — `try_embed_projection` produces it for `SELECT count(1) … JOIN …` (apache#20191) — and after a round-trip the join silently switches from "emit zero columns" to "emit all columns". `FilterExec` has a workaround for the same limitation; these two execs were missed. ## What changes are included in this PR? Encode `Some(vec![])` as the single-element sentinel `[u32::MAX]` (never a valid column index); recognise it on decode. Everything else goes through unchanged. ```rust // encode projection: match exec.projection.as_ref() { None => Vec::new(), Some(v) if v.is_empty() => vec![u32::MAX], Some(v) => v.iter().map(|x| *x as u32).collect(), }, // decode let projection = match hashjoin.projection.as_slice() { [] => None, [u32::MAX] => Some(Vec::new()), indices => Some(indices.iter().map(|i| *i as usize).collect()), }; ``` Applied symmetrically to `HashJoinExec` and `NestedLoopJoinExec`. ## Are these changes tested? yes, add roundtrip test case ## Are there any user-facing changes? <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> <!-- If there are any breaking changes to public APIs, please add the `api change` label. -->
1 parent b295fcf commit ea7ae7e

2 files changed

Lines changed: 84 additions & 25 deletions

File tree

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 31 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1400,16 +1400,15 @@ impl protobuf::PhysicalPlanNode {
14001400
protobuf::PartitionMode::Partitioned => PartitionMode::Partitioned,
14011401
protobuf::PartitionMode::Auto => PartitionMode::Auto,
14021402
};
1403-
let projection = if !hashjoin.projection.is_empty() {
1404-
Some(
1405-
hashjoin
1406-
.projection
1407-
.iter()
1408-
.map(|i| *i as usize)
1409-
.collect::<Vec<_>>(),
1410-
)
1411-
} else {
1412-
None
1403+
// Proto3 `repeated` cannot distinguish `None` from `Some(vec![])`. The latter
1404+
// is reachable via `try_embed_projection` for `SELECT count(1) … JOIN …` and
1405+
// changes the join's output schema, so the encoder reserves the single-element
1406+
// sentinel `[u32::MAX]` (never a valid column index) to mean "explicitly empty";
1407+
// every other state is sent as-is. See `try_from_hash_join_exec`.
1408+
let projection = match hashjoin.projection.as_slice() {
1409+
[] => None,
1410+
[u32::MAX] => Some(Vec::new()),
1411+
indices => Some(indices.iter().map(|i| *i as usize).collect()),
14131412
};
14141413
Ok(Arc::new(HashJoinExec::try_new(
14151414
left,
@@ -1804,15 +1803,13 @@ impl protobuf::PhysicalPlanNode {
18041803
})
18051804
.map_or(Ok(None), |v: Result<JoinFilter>| v.map(Some))?;
18061805

1807-
let projection = if !join.projection.is_empty() {
1808-
Some(
1809-
join.projection
1810-
.iter()
1811-
.map(|i| *i as usize)
1812-
.collect::<Vec<_>>(),
1813-
)
1814-
} else {
1815-
None
1806+
// See `try_into_hash_join_physical_plan` for the rationale behind the
1807+
// `[u32::MAX]` sentinel; `NestedLoopJoinExec` has the same `Option<Vec<usize>>`
1808+
// projection field and shares the proto3 `repeated` ambiguity.
1809+
let projection = match join.projection.as_slice() {
1810+
[] => None,
1811+
[u32::MAX] => Some(Vec::new()),
1812+
indices => Some(indices.iter().map(|i| *i as usize).collect()),
18161813
};
18171814

18181815
Ok(Arc::new(NestedLoopJoinExec::try_new(
@@ -2465,9 +2462,14 @@ impl protobuf::PhysicalPlanNode {
24652462
partition_mode: partition_mode.into(),
24662463
null_equality: null_equality.into(),
24672464
filter,
2468-
projection: exec.projection.as_ref().map_or_else(Vec::new, |v| {
2469-
v.iter().map(|x| *x as u32).collect::<Vec<u32>>()
2470-
}),
2465+
// Send `Some(vec![])` as `[u32::MAX]` (never a valid index) so the
2466+
// wire format can distinguish it from `None` (which stays empty).
2467+
// See `try_into_hash_join_physical_plan` for the matching decoder.
2468+
projection: match exec.projection.as_ref() {
2469+
None => Vec::new(),
2470+
Some(v) if v.is_empty() => vec![u32::MAX],
2471+
Some(v) => v.iter().map(|x| *x as u32).collect(),
2472+
},
24712473
null_aware: exec.null_aware,
24722474
},
24732475
))),
@@ -3239,9 +3241,13 @@ impl protobuf::PhysicalPlanNode {
32393241
right: Some(Box::new(right)),
32403242
join_type: join_type.into(),
32413243
filter,
3242-
projection: exec.projection().as_ref().map_or_else(Vec::new, |v| {
3243-
v.iter().map(|x| *x as u32).collect::<Vec<u32>>()
3244-
}),
3244+
// `[u32::MAX]` sentinel distinguishes `Some(vec![])` from `None`;
3245+
// see `try_from_hash_join_exec`.
3246+
projection: match exec.projection().as_ref() {
3247+
None => Vec::new(),
3248+
Some(v) if v.is_empty() => vec![u32::MAX],
3249+
Some(v) => v.iter().map(|x| *x as u32).collect(),
3250+
},
32453251
},
32463252
))),
32473253
})

‎datafusion/proto/tests/cases/roundtrip_physical_plan.rs‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,59 @@ fn roundtrip_nested_loop_join() -> Result<()> {
338338
Ok(())
339339
}
340340

341+
/// Regression: proto3 `repeated` fields cannot distinguish "absent" from "empty",
342+
/// so a naive encoding collapses `Some(vec![])` and `None` into the same wire
343+
/// representation. `try_embed_projection` (DataFusion 53+) produces
344+
/// `HashJoinExec.projection = Some(vec![])` for `SELECT count(1) … JOIN …`,
345+
/// which previously round-tripped to `None` and caused downstream consumers (e.g.
346+
/// distributed Flight executors) to receive a different number of output
347+
/// columns than the planner declared. Verify all three states preserve.
348+
#[test]
349+
fn roundtrip_hash_join_projection_states() -> Result<()> {
350+
let field_a = Field::new("col", DataType::Int64, false);
351+
let schema_left = Arc::new(Schema::new(vec![field_a.clone()]));
352+
let schema_right = Arc::new(Schema::new(vec![field_a]));
353+
let on = vec![(
354+
Arc::new(Column::new("col", schema_left.index_of("col")?)) as _,
355+
Arc::new(Column::new("col", schema_right.index_of("col")?)) as _,
356+
)];
357+
358+
for projection in [None, Some(vec![]), Some(vec![0]), Some(vec![1])] {
359+
roundtrip_test(Arc::new(HashJoinExec::try_new(
360+
Arc::new(EmptyExec::new(schema_left.clone())),
361+
Arc::new(EmptyExec::new(schema_right.clone())),
362+
on.clone(),
363+
None,
364+
&JoinType::Inner,
365+
projection,
366+
PartitionMode::Partitioned,
367+
NullEquality::NullEqualsNothing,
368+
false,
369+
)?))?;
370+
}
371+
Ok(())
372+
}
373+
374+
/// Same regression coverage for `NestedLoopJoinExec`, which shares the
375+
/// `repeated uint32 projection` proto field shape with `HashJoinExec`.
376+
#[test]
377+
fn roundtrip_nested_loop_join_projection_states() -> Result<()> {
378+
let field_a = Field::new("col", DataType::Int64, false);
379+
let schema_left = Arc::new(Schema::new(vec![field_a.clone()]));
380+
let schema_right = Arc::new(Schema::new(vec![field_a]));
381+
382+
for projection in [None, Some(vec![]), Some(vec![0]), Some(vec![1])] {
383+
roundtrip_test(Arc::new(NestedLoopJoinExec::try_new(
384+
Arc::new(EmptyExec::new(schema_left.clone())),
385+
Arc::new(EmptyExec::new(schema_right.clone())),
386+
None,
387+
&JoinType::Inner,
388+
projection,
389+
)?))?;
390+
}
391+
Ok(())
392+
}
393+
341394
#[test]
342395
fn roundtrip_udwf() -> Result<()> {
343396
let field_a = Field::new("a", DataType::Int64, false);

0 commit comments

Comments
 (0)