Skip to content

Commit e2e161d

Browse files
adriangbclaude
andauthored
test(physical-plan): add colocated field-level plan serde tests (apache#24222)
Adds the second, colocated test tier proposed in apache#24171 (d), next to the serde hooks the plans already own. `datafusion-physical-plan` gains `proto_test_util`, the plan-level sibling of `datafusion_physical_expr::proto_test_util`: stub `ExecutionPlanEncode` / `ExecutionPlanDecode` implementations a test supplies itself, so a plan's `try_to_proto` / `try_from_proto` can be driven without depending on `datafusion-proto` (which sits above this crate). The stubs count their calls and can fail on the Nth one, so the `?` arms are covered too. On top of that, colocated `proto_tests` modules for the fields where being wrong is expensive and `Debug`-comparing round-trip tests are blind: * `SortExec` / `SortPreservingMergeExec` — `fetch` presence semantics (absent -> `None`, never `Some(0)`), `preserve_partitioning`, the `asc`/`descending` inversion, the TopK dynamic filter, and the reject paths. `SortPreservingMergeExec` also pins that `enable_round_robin_repartition` is deliberately not on the wire. * `HashJoinExec` — the three `projection` states proto3 cannot express directly (the `[u32::MAX]` sentinel), `fetch` presence (the field dropped in apache#24165), the checked `u64` -> `usize` conversion, and `PartitionMode`. * `NestedLoopJoinExec` — the same projection sentinel, which is written out a second time in that file and so is tested a second time here. * `joins::proto` — exhaustive by-name round trips for `JoinType`, `JoinSide` and `NullEquality`, plus a guard that the two numberings really do differ, so nobody "simplifies" the matches into a cast. Per apache#24171 (c) the new tests were verified against a deliberately broken encode side rather than assumed to bite. This supplements the central round-trip tests, it does not replace them: those still prove the real `PhysicalExtensionCodec` works, that dispatch reaches the hook, and that bytes survive bytes. --------- Co-authored-by: Claude <[email protected]>
1 parent c429919 commit e2e161d

7 files changed

Lines changed: 1455 additions & 0 deletions

File tree

‎datafusion/physical-plan/src/joins/hash_join/exec.rs‎

Lines changed: 291 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2062,6 +2062,297 @@ impl HashJoinExec {
20622062
}
20632063
}
20642064

2065+
/// Field-level tests for the `try_to_proto` / `try_from_proto` hooks.
2066+
///
2067+
/// These cover the three states of `projection` that proto3 cannot express
2068+
/// directly (the `[u32::MAX]` sentinel), `fetch` presence semantics — the field
2069+
/// dropped in #24165, which the central `Debug`-comparing round-trip tests
2070+
/// could not see — and the by-name `PartitionMode` mapping, whose discriminants
2071+
/// deliberately differ between the plan and the wire.
2072+
#[cfg(all(test, feature = "proto"))]
2073+
mod proto_tests {
2074+
use super::*;
2075+
use crate::proto::{ExecutionPlanDecodeCtx, ExecutionPlanEncodeCtx};
2076+
use crate::proto_test_util::{
2077+
StubPlanDecoder, StubPlanEncoder, UnreachablePlanDecoder, column_node,
2078+
encoded_child_node, stub_child,
2079+
};
2080+
use datafusion_physical_expr::expressions::Column;
2081+
use datafusion_proto_models::protobuf;
2082+
2083+
/// An inner hash join on `a = a` between two stub children.
2084+
fn join_builder() -> HashJoinExecBuilder {
2085+
let on: Vec<(PhysicalExprRef, PhysicalExprRef)> =
2086+
vec![(Arc::new(Column::new("a", 0)), Arc::new(Column::new("a", 0)))];
2087+
HashJoinExecBuilder::new(stub_child(), stub_child(), on, JoinType::Inner)
2088+
}
2089+
2090+
/// Encode `plan` with a stub encoder, returning the `HashJoinExecNode`.
2091+
fn encode(
2092+
plan: &HashJoinExec,
2093+
encoder: &StubPlanEncoder,
2094+
) -> protobuf::HashJoinExecNode {
2095+
let ctx = ExecutionPlanEncodeCtx::new(encoder);
2096+
let node = plan
2097+
.try_to_proto(&ctx)
2098+
.unwrap()
2099+
.expect("HashJoinExec should encode to Some(node)");
2100+
match node.physical_plan_type {
2101+
Some(protobuf::physical_plan_node::PhysicalPlanType::HashJoin(join)) => *join,
2102+
other => panic!("expected a HashJoin node, got {other:?}"),
2103+
}
2104+
}
2105+
2106+
/// Encode a join whose only non-default state is its projection.
2107+
fn encode_projection(projection: Option<Vec<usize>>) -> Vec<u32> {
2108+
let plan = join_builder().with_projection(projection).build().unwrap();
2109+
encode(&plan, &StubPlanEncoder::ok()).projection
2110+
}
2111+
2112+
/// A hand-built `HashJoinExecNode` wrapped in its `PhysicalPlanNode`.
2113+
fn join_node(node: protobuf::HashJoinExecNode) -> protobuf::PhysicalPlanNode {
2114+
protobuf::PhysicalPlanNode {
2115+
physical_plan_type: Some(
2116+
protobuf::physical_plan_node::PhysicalPlanType::HashJoin(Box::new(node)),
2117+
),
2118+
}
2119+
}
2120+
2121+
/// A decodable `HashJoinExecNode`: two children, one join key, no filter.
2122+
fn decodable_node() -> protobuf::HashJoinExecNode {
2123+
protobuf::HashJoinExecNode {
2124+
left: Some(Box::new(encoded_child_node())),
2125+
right: Some(Box::new(encoded_child_node())),
2126+
on: vec![protobuf::JoinOn {
2127+
left: Some(column_node("a", 0)),
2128+
right: Some(column_node("a", 0)),
2129+
}],
2130+
join_type: protobuf::JoinType::Inner.into(),
2131+
partition_mode: protobuf::PartitionMode::Partitioned.into(),
2132+
null_equality: protobuf::NullEquality::NullEqualsNothing.into(),
2133+
filter: None,
2134+
projection: vec![],
2135+
null_aware: false,
2136+
dynamic_filter: None,
2137+
fetch: None,
2138+
}
2139+
}
2140+
2141+
/// Decode `node` with a stub decoder.
2142+
fn decode(node: protobuf::HashJoinExecNode) -> Arc<dyn ExecutionPlan> {
2143+
let decoder = StubPlanDecoder::ok();
2144+
let ctx = ExecutionPlanDecodeCtx::new(&decoder);
2145+
HashJoinExec::try_from_proto(&join_node(node), &ctx).unwrap()
2146+
}
2147+
2148+
/// View a decoded plan as the `HashJoinExec` it must be.
2149+
fn as_join(plan: &Arc<dyn ExecutionPlan>) -> &HashJoinExec {
2150+
plan.downcast_ref::<HashJoinExec>()
2151+
.expect("decoded plan should be a HashJoinExec")
2152+
}
2153+
2154+
/// "No projection" is the empty repeated field.
2155+
#[test]
2156+
fn try_to_proto_encodes_an_absent_projection_as_empty() {
2157+
assert_eq!(encode_projection(None), Vec::<u32>::new());
2158+
}
2159+
2160+
/// An *empty* projection changes the output schema, so it cannot share the
2161+
/// "absent" encoding: it goes out as the `[u32::MAX]` sentinel.
2162+
#[test]
2163+
fn try_to_proto_encodes_an_empty_projection_as_the_sentinel() {
2164+
assert_eq!(encode_projection(Some(vec![])), vec![u32::MAX]);
2165+
}
2166+
2167+
#[test]
2168+
fn try_to_proto_encodes_a_non_empty_projection_as_is() {
2169+
assert_eq!(encode_projection(Some(vec![0, 2])), vec![0, 2]);
2170+
}
2171+
2172+
#[test]
2173+
fn try_from_proto_decodes_an_empty_projection_field_as_absent() {
2174+
let plan = decode(decodable_node());
2175+
2176+
assert!(as_join(&plan).projection.is_none());
2177+
}
2178+
2179+
#[test]
2180+
fn try_from_proto_decodes_the_sentinel_as_an_empty_projection() {
2181+
let mut node = decodable_node();
2182+
node.projection = vec![u32::MAX];
2183+
2184+
let plan = decode(node);
2185+
let projection = as_join(&plan)
2186+
.projection
2187+
.as_ref()
2188+
.expect("the sentinel decodes to Some(empty)");
2189+
assert!(projection.is_empty());
2190+
}
2191+
2192+
#[test]
2193+
fn try_from_proto_decodes_a_non_empty_projection_as_is() {
2194+
let mut node = decodable_node();
2195+
node.projection = vec![0, 2];
2196+
2197+
let plan = decode(node);
2198+
assert_eq!(
2199+
as_join(&plan).projection.as_deref(),
2200+
Some([0, 2].as_slice())
2201+
);
2202+
}
2203+
2204+
/// The regression guard for #24165: an unlimited join must not encode as
2205+
/// `Some(0)`, and a limited one must keep its limit.
2206+
#[test]
2207+
fn try_to_proto_encodes_fetch_by_presence() {
2208+
let unlimited = join_builder().build().unwrap();
2209+
assert_eq!(encode(&unlimited, &StubPlanEncoder::ok()).fetch, None);
2210+
2211+
let limited = join_builder().with_fetch(Some(10)).build().unwrap();
2212+
assert_eq!(encode(&limited, &StubPlanEncoder::ok()).fetch, Some(10));
2213+
2214+
let empty = join_builder().with_fetch(Some(0)).build().unwrap();
2215+
assert_eq!(encode(&empty, &StubPlanEncoder::ok()).fetch, Some(0));
2216+
}
2217+
2218+
/// A message written before `fetch` existed has no value on the wire, and
2219+
/// must decode to "no limit" rather than to `Some(0)` ("no rows").
2220+
#[test]
2221+
fn try_from_proto_decodes_an_absent_fetch_as_no_limit() {
2222+
assert_eq!(as_join(&decode(decodable_node())).fetch, None);
2223+
}
2224+
2225+
#[test]
2226+
fn try_from_proto_decodes_a_present_fetch() {
2227+
let mut node = decodable_node();
2228+
node.fetch = Some(10);
2229+
assert_eq!(as_join(&decode(node)).fetch, Some(10));
2230+
2231+
let mut node = decodable_node();
2232+
node.fetch = Some(0);
2233+
assert_eq!(as_join(&decode(node)).fetch, Some(0));
2234+
}
2235+
2236+
/// `fetch` is a `u64` on the wire and a `usize` in the plan. The conversion
2237+
/// is checked rather than `as usize`, so a value too large for the target
2238+
/// is reported instead of truncated — on a 64-bit target every `u64` fits,
2239+
/// and the largest one must come back intact rather than wrapping.
2240+
#[test]
2241+
#[cfg(target_pointer_width = "64")]
2242+
fn try_from_proto_decodes_the_largest_fetch_without_truncating() {
2243+
let plan = decode({
2244+
let mut node = decodable_node();
2245+
node.fetch = Some(u64::MAX);
2246+
node
2247+
});
2248+
2249+
assert_eq!(as_join(&plan).fetch, Some(usize::MAX));
2250+
}
2251+
2252+
/// The plan-side and wire-side `PartitionMode` discriminants differ, so the
2253+
/// mapping has to be by name in both directions.
2254+
#[test]
2255+
fn partition_mode_round_trips_by_name() {
2256+
for (mode, wire) in [
2257+
(
2258+
PartitionMode::CollectLeft,
2259+
protobuf::PartitionMode::CollectLeft,
2260+
),
2261+
(
2262+
PartitionMode::Partitioned,
2263+
protobuf::PartitionMode::Partitioned,
2264+
),
2265+
(PartitionMode::Auto, protobuf::PartitionMode::Auto),
2266+
] {
2267+
let plan = join_builder().with_partition_mode(mode).build().unwrap();
2268+
let node = encode(&plan, &StubPlanEncoder::ok());
2269+
assert_eq!(node.partition_mode, i32::from(wire), "encoding {mode:?}");
2270+
2271+
let mut decodable = decodable_node();
2272+
decodable.partition_mode = node.partition_mode;
2273+
assert_eq!(*as_join(&decode(decodable)).partition_mode(), mode);
2274+
}
2275+
}
2276+
2277+
#[test]
2278+
fn try_from_proto_rejects_an_unknown_partition_mode() {
2279+
let decoder = StubPlanDecoder::ok();
2280+
let ctx = ExecutionPlanDecodeCtx::new(&decoder);
2281+
let mut node = decodable_node();
2282+
node.partition_mode = 42;
2283+
2284+
let err = HashJoinExec::try_from_proto(&join_node(node), &ctx).unwrap_err();
2285+
assert!(
2286+
err.to_string()
2287+
.contains("HashJoinExec: unknown PartitionMode 42")
2288+
);
2289+
}
2290+
2291+
/// `null_aware` is only legal on a `LeftAnti` join, so the round trip has
2292+
/// to carry the join type with it.
2293+
#[test]
2294+
fn null_aware_round_trips() {
2295+
let plan = join_builder()
2296+
.with_type(JoinType::LeftAnti)
2297+
.with_null_aware(true)
2298+
.build()
2299+
.unwrap();
2300+
assert!(encode(&plan, &StubPlanEncoder::ok()).null_aware);
2301+
2302+
let mut node = decodable_node();
2303+
node.join_type = protobuf::JoinType::Leftanti.into();
2304+
node.null_aware = true;
2305+
assert!(as_join(&decode(node)).null_aware);
2306+
}
2307+
2308+
#[test]
2309+
fn try_to_proto_encodes_both_children_and_both_key_sides() {
2310+
let encoder = StubPlanEncoder::ok();
2311+
let node = encode(&join_builder().build().unwrap(), &encoder);
2312+
2313+
assert_eq!(encoder.plan_calls(), 2);
2314+
assert_eq!(encoder.expr_calls(), 2);
2315+
assert_eq!(node.left, Some(Box::new(encoded_child_node())));
2316+
assert_eq!(node.right, Some(Box::new(encoded_child_node())));
2317+
}
2318+
2319+
#[test]
2320+
fn try_to_proto_propagates_child_encode_error() {
2321+
let plan = join_builder().build().unwrap();
2322+
let encoder = StubPlanEncoder::failing_on_plan(2);
2323+
let ctx = ExecutionPlanEncodeCtx::new(&encoder);
2324+
2325+
let err = plan.try_to_proto(&ctx).unwrap_err();
2326+
assert!(
2327+
err.to_string()
2328+
.contains("stub plan encode failure on call 2")
2329+
);
2330+
}
2331+
2332+
#[test]
2333+
fn try_from_proto_rejects_a_different_plan_variant() {
2334+
let decoder = UnreachablePlanDecoder::new();
2335+
let ctx = ExecutionPlanDecodeCtx::new(&decoder);
2336+
2337+
let err = HashJoinExec::try_from_proto(&encoded_child_node(), &ctx).unwrap_err();
2338+
assert!(err.to_string().contains("not a HashJoinExec"));
2339+
}
2340+
2341+
#[test]
2342+
fn try_from_proto_rejects_a_missing_child() {
2343+
let decoder = UnreachablePlanDecoder::new();
2344+
let ctx = ExecutionPlanDecodeCtx::new(&decoder);
2345+
let mut node = decodable_node();
2346+
node.left = None;
2347+
2348+
let err = HashJoinExec::try_from_proto(&join_node(node), &ctx).unwrap_err();
2349+
assert!(
2350+
err.to_string()
2351+
.contains("HashJoinExec is missing required field 'left'")
2352+
);
2353+
}
2354+
}
2355+
20652356
/// Determines which sides of a join are "preserved" for filter pushdown.
20662357
///
20672358
/// A preserved side means filters on that side's columns can be safely pushed

0 commit comments

Comments
 (0)