Skip to content

Commit ef93155

Browse files
committed
test(physical-plan): add colocated field-level plan serde tests
Adds the second, colocated test tier proposed in apache#24171 (d), next to the serde hooks the plans already own. `datafusion-physical-plan` gains `proto_test_util`, the plan-level sibling of `datafusion_physical_expr::proto_test_util`: stub `ExecutionPlanEncode` / `ExecutionPlanDecode` implementations a test supplies itself, so a plan's `try_to_proto` / `try_from_proto` can be driven without depending on `datafusion-proto` (which sits above this crate). The stubs count their calls and can fail on the Nth one, so the `?` arms are covered too. On top of that, colocated `proto_tests` modules for the fields where being wrong is expensive and `Debug`-comparing round-trip tests are blind: * `SortExec` / `SortPreservingMergeExec` — `fetch` presence semantics (absent -> `None`, never `Some(0)`), `preserve_partitioning`, the `asc`/`descending` inversion, the TopK dynamic filter, and the reject paths. `SortPreservingMergeExec` also pins that `enable_round_robin_repartition` is deliberately not on the wire. * `HashJoinExec` — the three `projection` states proto3 cannot express directly (the `[u32::MAX]` sentinel), `fetch` presence (the field dropped in apache#24165), the checked `u64` -> `usize` conversion, and `PartitionMode`. * `NestedLoopJoinExec` — the same projection sentinel, which is written out a second time in that file and so is tested a second time here. * `joins::proto` — exhaustive by-name round trips for `JoinType`, `JoinSide` and `NullEquality`, plus a guard that the two numberings really do differ, so nobody "simplifies" the matches into a cast. Per apache#24171 (c) the new tests were verified against a deliberately broken encode side rather than assumed to bite. This supplements the central round-trip tests, it does not replace them: those still prove the real `PhysicalExtensionCodec` works, that dispatch reaches the hook, and that bytes survive bytes.
1 parent f4c8ba1 commit ef93155

7 files changed

Lines changed: 1398 additions & 0 deletions

File tree

‎datafusion/physical-plan/src/joins/hash_join/exec.rs‎

Lines changed: 291 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2008,6 +2008,297 @@ impl HashJoinExec {
20082008
}
20092009
}
20102010

2011+
/// Field-level tests for the `try_to_proto` / `try_from_proto` hooks.
2012+
///
2013+
/// These cover the three states of `projection` that proto3 cannot express
2014+
/// directly (the `[u32::MAX]` sentinel), `fetch` presence semantics — the field
2015+
/// dropped in #24165, which the central `Debug`-comparing round-trip tests
2016+
/// could not see — and the by-name `PartitionMode` mapping, whose discriminants
2017+
/// deliberately differ between the plan and the wire.
2018+
#[cfg(all(test, feature = "proto"))]
2019+
mod proto_tests {
2020+
use super::*;
2021+
use crate::proto::{ExecutionPlanDecodeCtx, ExecutionPlanEncodeCtx};
2022+
use crate::proto_test_util::{
2023+
StubPlanDecoder, StubPlanEncoder, UnreachablePlanDecoder, column_node,
2024+
encoded_child_node, stub_child,
2025+
};
2026+
use datafusion_physical_expr::expressions::Column;
2027+
use datafusion_proto_models::protobuf;
2028+
2029+
/// An inner hash join on `a = a` between two stub children.
2030+
fn join_builder() -> HashJoinExecBuilder {
2031+
let on: Vec<(PhysicalExprRef, PhysicalExprRef)> =
2032+
vec![(Arc::new(Column::new("a", 0)), Arc::new(Column::new("a", 0)))];
2033+
HashJoinExecBuilder::new(stub_child(), stub_child(), on, JoinType::Inner)
2034+
}
2035+
2036+
/// Encode `plan` with a stub encoder, returning the `HashJoinExecNode`.
2037+
fn encode(
2038+
plan: &HashJoinExec,
2039+
encoder: &StubPlanEncoder,
2040+
) -> protobuf::HashJoinExecNode {
2041+
let ctx = ExecutionPlanEncodeCtx::new(encoder);
2042+
let node = plan
2043+
.try_to_proto(&ctx)
2044+
.unwrap()
2045+
.expect("HashJoinExec should encode to Some(node)");
2046+
match node.physical_plan_type {
2047+
Some(protobuf::physical_plan_node::PhysicalPlanType::HashJoin(join)) => *join,
2048+
other => panic!("expected a HashJoin node, got {other:?}"),
2049+
}
2050+
}
2051+
2052+
/// Encode a join whose only non-default state is its projection.
2053+
fn encode_projection(projection: Option<Vec<usize>>) -> Vec<u32> {
2054+
let plan = join_builder().with_projection(projection).build().unwrap();
2055+
encode(&plan, &StubPlanEncoder::ok()).projection
2056+
}
2057+
2058+
/// A hand-built `HashJoinExecNode` wrapped in its `PhysicalPlanNode`.
2059+
fn join_node(node: protobuf::HashJoinExecNode) -> protobuf::PhysicalPlanNode {
2060+
protobuf::PhysicalPlanNode {
2061+
physical_plan_type: Some(
2062+
protobuf::physical_plan_node::PhysicalPlanType::HashJoin(Box::new(node)),
2063+
),
2064+
}
2065+
}
2066+
2067+
/// A decodable `HashJoinExecNode`: two children, one join key, no filter.
2068+
fn decodable_node() -> protobuf::HashJoinExecNode {
2069+
protobuf::HashJoinExecNode {
2070+
left: Some(Box::new(encoded_child_node())),
2071+
right: Some(Box::new(encoded_child_node())),
2072+
on: vec![protobuf::JoinOn {
2073+
left: Some(column_node("a", 0)),
2074+
right: Some(column_node("a", 0)),
2075+
}],
2076+
join_type: protobuf::JoinType::Inner.into(),
2077+
partition_mode: protobuf::PartitionMode::Partitioned.into(),
2078+
null_equality: protobuf::NullEquality::NullEqualsNothing.into(),
2079+
filter: None,
2080+
projection: vec![],
2081+
null_aware: false,
2082+
dynamic_filter: None,
2083+
fetch: None,
2084+
}
2085+
}
2086+
2087+
/// Decode `node` with a stub decoder.
2088+
fn decode(node: protobuf::HashJoinExecNode) -> Arc<dyn ExecutionPlan> {
2089+
let decoder = StubPlanDecoder::ok();
2090+
let ctx = ExecutionPlanDecodeCtx::new(&decoder);
2091+
HashJoinExec::try_from_proto(&join_node(node), &ctx).unwrap()
2092+
}
2093+
2094+
/// View a decoded plan as the `HashJoinExec` it must be.
2095+
fn as_join(plan: &Arc<dyn ExecutionPlan>) -> &HashJoinExec {
2096+
plan.downcast_ref::<HashJoinExec>()
2097+
.expect("decoded plan should be a HashJoinExec")
2098+
}
2099+
2100+
/// "No projection" is the empty repeated field.
2101+
#[test]
2102+
fn try_to_proto_encodes_an_absent_projection_as_empty() {
2103+
assert_eq!(encode_projection(None), Vec::<u32>::new());
2104+
}
2105+
2106+
/// An *empty* projection changes the output schema, so it cannot share the
2107+
/// "absent" encoding: it goes out as the `[u32::MAX]` sentinel.
2108+
#[test]
2109+
fn try_to_proto_encodes_an_empty_projection_as_the_sentinel() {
2110+
assert_eq!(encode_projection(Some(vec![])), vec![u32::MAX]);
2111+
}
2112+
2113+
#[test]
2114+
fn try_to_proto_encodes_a_non_empty_projection_as_is() {
2115+
assert_eq!(encode_projection(Some(vec![0, 2])), vec![0, 2]);
2116+
}
2117+
2118+
#[test]
2119+
fn try_from_proto_decodes_an_empty_projection_field_as_absent() {
2120+
let plan = decode(decodable_node());
2121+
2122+
assert!(as_join(&plan).projection.is_none());
2123+
}
2124+
2125+
#[test]
2126+
fn try_from_proto_decodes_the_sentinel_as_an_empty_projection() {
2127+
let mut node = decodable_node();
2128+
node.projection = vec![u32::MAX];
2129+
2130+
let plan = decode(node);
2131+
let projection = as_join(&plan)
2132+
.projection
2133+
.as_ref()
2134+
.expect("the sentinel decodes to Some(empty)");
2135+
assert!(projection.is_empty());
2136+
}
2137+
2138+
#[test]
2139+
fn try_from_proto_decodes_a_non_empty_projection_as_is() {
2140+
let mut node = decodable_node();
2141+
node.projection = vec![0, 2];
2142+
2143+
let plan = decode(node);
2144+
assert_eq!(
2145+
as_join(&plan).projection.as_deref(),
2146+
Some([0, 2].as_slice())
2147+
);
2148+
}
2149+
2150+
/// The regression guard for #24165: an unlimited join must not encode as
2151+
/// `Some(0)`, and a limited one must keep its limit.
2152+
#[test]
2153+
fn try_to_proto_encodes_fetch_by_presence() {
2154+
let unlimited = join_builder().build().unwrap();
2155+
assert_eq!(encode(&unlimited, &StubPlanEncoder::ok()).fetch, None);
2156+
2157+
let limited = join_builder().with_fetch(Some(10)).build().unwrap();
2158+
assert_eq!(encode(&limited, &StubPlanEncoder::ok()).fetch, Some(10));
2159+
2160+
let empty = join_builder().with_fetch(Some(0)).build().unwrap();
2161+
assert_eq!(encode(&empty, &StubPlanEncoder::ok()).fetch, Some(0));
2162+
}
2163+
2164+
/// A message written before `fetch` existed has no value on the wire, and
2165+
/// must decode to "no limit" rather than to `Some(0)` ("no rows").
2166+
#[test]
2167+
fn try_from_proto_decodes_an_absent_fetch_as_no_limit() {
2168+
assert_eq!(as_join(&decode(decodable_node())).fetch, None);
2169+
}
2170+
2171+
#[test]
2172+
fn try_from_proto_decodes_a_present_fetch() {
2173+
let mut node = decodable_node();
2174+
node.fetch = Some(10);
2175+
assert_eq!(as_join(&decode(node)).fetch, Some(10));
2176+
2177+
let mut node = decodable_node();
2178+
node.fetch = Some(0);
2179+
assert_eq!(as_join(&decode(node)).fetch, Some(0));
2180+
}
2181+
2182+
/// `fetch` is a `u64` on the wire and a `usize` in the plan. The conversion
2183+
/// is checked rather than `as usize`, so a value too large for the target
2184+
/// is reported instead of truncated — on a 64-bit target every `u64` fits,
2185+
/// and the largest one must come back intact rather than wrapping.
2186+
#[test]
2187+
#[cfg(target_pointer_width = "64")]
2188+
fn try_from_proto_decodes_the_largest_fetch_without_truncating() {
2189+
let plan = decode({
2190+
let mut node = decodable_node();
2191+
node.fetch = Some(u64::MAX);
2192+
node
2193+
});
2194+
2195+
assert_eq!(as_join(&plan).fetch, Some(usize::MAX));
2196+
}
2197+
2198+
/// The plan-side and wire-side `PartitionMode` discriminants differ, so the
2199+
/// mapping has to be by name in both directions.
2200+
#[test]
2201+
fn partition_mode_round_trips_by_name() {
2202+
for (mode, wire) in [
2203+
(
2204+
PartitionMode::CollectLeft,
2205+
protobuf::PartitionMode::CollectLeft,
2206+
),
2207+
(
2208+
PartitionMode::Partitioned,
2209+
protobuf::PartitionMode::Partitioned,
2210+
),
2211+
(PartitionMode::Auto, protobuf::PartitionMode::Auto),
2212+
] {
2213+
let plan = join_builder().with_partition_mode(mode).build().unwrap();
2214+
let node = encode(&plan, &StubPlanEncoder::ok());
2215+
assert_eq!(node.partition_mode, i32::from(wire), "encoding {mode:?}");
2216+
2217+
let mut decodable = decodable_node();
2218+
decodable.partition_mode = node.partition_mode;
2219+
assert_eq!(*as_join(&decode(decodable)).partition_mode(), mode);
2220+
}
2221+
}
2222+
2223+
#[test]
2224+
fn try_from_proto_rejects_an_unknown_partition_mode() {
2225+
let decoder = StubPlanDecoder::ok();
2226+
let ctx = ExecutionPlanDecodeCtx::new(&decoder);
2227+
let mut node = decodable_node();
2228+
node.partition_mode = 42;
2229+
2230+
let err = HashJoinExec::try_from_proto(&join_node(node), &ctx).unwrap_err();
2231+
assert!(
2232+
err.to_string()
2233+
.contains("HashJoinExec: unknown PartitionMode 42")
2234+
);
2235+
}
2236+
2237+
/// `null_aware` is only legal on a `LeftAnti` join, so the round trip has
2238+
/// to carry the join type with it.
2239+
#[test]
2240+
fn null_aware_round_trips() {
2241+
let plan = join_builder()
2242+
.with_type(JoinType::LeftAnti)
2243+
.with_null_aware(true)
2244+
.build()
2245+
.unwrap();
2246+
assert!(encode(&plan, &StubPlanEncoder::ok()).null_aware);
2247+
2248+
let mut node = decodable_node();
2249+
node.join_type = protobuf::JoinType::Leftanti.into();
2250+
node.null_aware = true;
2251+
assert!(as_join(&decode(node)).null_aware);
2252+
}
2253+
2254+
#[test]
2255+
fn try_to_proto_encodes_both_children_and_both_key_sides() {
2256+
let encoder = StubPlanEncoder::ok();
2257+
let node = encode(&join_builder().build().unwrap(), &encoder);
2258+
2259+
assert_eq!(encoder.plan_calls(), 2);
2260+
assert_eq!(encoder.expr_calls(), 2);
2261+
assert_eq!(node.left, Some(Box::new(encoded_child_node())));
2262+
assert_eq!(node.right, Some(Box::new(encoded_child_node())));
2263+
}
2264+
2265+
#[test]
2266+
fn try_to_proto_propagates_child_encode_error() {
2267+
let plan = join_builder().build().unwrap();
2268+
let encoder = StubPlanEncoder::failing_on_plan(2);
2269+
let ctx = ExecutionPlanEncodeCtx::new(&encoder);
2270+
2271+
let err = plan.try_to_proto(&ctx).unwrap_err();
2272+
assert!(
2273+
err.to_string()
2274+
.contains("stub plan encode failure on call 2")
2275+
);
2276+
}
2277+
2278+
#[test]
2279+
fn try_from_proto_rejects_a_different_plan_variant() {
2280+
let decoder = UnreachablePlanDecoder::new();
2281+
let ctx = ExecutionPlanDecodeCtx::new(&decoder);
2282+
2283+
let err = HashJoinExec::try_from_proto(&encoded_child_node(), &ctx).unwrap_err();
2284+
assert!(err.to_string().contains("not a HashJoinExec"));
2285+
}
2286+
2287+
#[test]
2288+
fn try_from_proto_rejects_a_missing_child() {
2289+
let decoder = UnreachablePlanDecoder::new();
2290+
let ctx = ExecutionPlanDecodeCtx::new(&decoder);
2291+
let mut node = decodable_node();
2292+
node.left = None;
2293+
2294+
let err = HashJoinExec::try_from_proto(&join_node(node), &ctx).unwrap_err();
2295+
assert!(
2296+
err.to_string()
2297+
.contains("HashJoinExec is missing required field 'left'")
2298+
);
2299+
}
2300+
}
2301+
20112302
/// Determines which sides of a join are "preserved" for filter pushdown.
20122303
///
20132304
/// A preserved side means filters on that side's columns can be safely pushed

0 commit comments

Comments
 (0)