Skip to content

Commit 81cdb1a

Browse files
nanookclawadriangb
andauthored
refactor: port HashExpr proto hooks (apache#22502)
## Which issue does this PR close? - Closes apache#22432. ## Rationale for this change `HashExpr` is part of the physical expression proto cleanup tracked by apache#22418. Its protobuf serialization should live with the expression implementation instead of in the central proto downcast chains, matching the hook pattern added in apache#21929. ## What changes are included in this PR? This adds `HashExpr`'s `PhysicalExpr::try_to_proto` override and a feature-gated inherent `HashExpr::try_from_proto`. The existing `PhysicalHashExprNode` wire shape is preserved, including `expr_id: None`, while `datafusion-proto` now routes `ExprType::HashExpr` decoding through the hook and no longer has a central `HashExpr` serialization arm. `datafusion-physical-plan` now exposes a `proto` feature so the hook code only compiles when proto support is requested. ## Are these changes tested? Yes. Added focused direct hook tests for encoding, decoding, and rejecting a wrong `expr_type`. Commands run: - `cargo fmt --all -- --check` - `cargo check -p datafusion-physical-plan` - `cargo test -p datafusion-physical-plan --features proto hash_expr_try` - `cargo test -p datafusion-proto --test proto_integration cases::roundtrip_physical_plan::roundtrip_hash_expr` - `cargo check -p datafusion-proto` - `cargo clippy -p datafusion-physical-plan --features proto --all-targets -- -D warnings` - `cargo clippy -p datafusion-proto --all-targets -- -D warnings` ## Are there any user-facing changes? No. This preserves the existing protobuf representation. --------- Signed-off-by: Nanook Claw <[email protected]> Signed-off-by: Nanook <[email protected]> Co-authored-by: Nanook <[email protected]> Co-authored-by: Adrian Garcia Badaracco <[email protected]>
1 parent cab69a1 commit 81cdb1a

4 files changed

Lines changed: 223 additions & 30 deletions

File tree

‎datafusion/physical-plan/Cargo.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,13 @@ force_hash_collisions = []
4242
test_utils = ["arrow/test_utils"]
4343
tokio_coop = []
4444
tokio_coop_fallback = []
45+
# Enables `PhysicalExpr::try_to_proto` / `try_from_proto` hooks on the
46+
# physical expressions defined in this crate (e.g. `HashExpr`). Off by
47+
# default so consumers that never serialize plans pay nothing.
4548
proto = [
4649
"dep:datafusion-proto-models",
4750
"dep:datafusion-proto-common",
51+
"datafusion-physical-expr/proto",
4852
"datafusion-physical-expr-common/proto",
4953
]
5054

‎datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs‎

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ use arrow::{
2727
use datafusion_common::Result;
2828
use datafusion_common::hash_utils::RandomState;
2929
use datafusion_common::hash_utils::{create_hashes, with_hashes};
30+
#[cfg(feature = "proto")]
31+
use datafusion_common::internal_err;
3032
use datafusion_expr::ColumnarValue;
3133
use datafusion_physical_expr_common::physical_expr::{
3234
DynHash, PhysicalExpr, PhysicalExprRef,
@@ -199,6 +201,55 @@ impl PhysicalExpr for HashExpr {
199201
fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
200202
write!(f, "{}", self.description)
201203
}
204+
205+
#[cfg(feature = "proto")]
206+
fn try_to_proto(
207+
&self,
208+
ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,
209+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {
210+
use datafusion_proto_models::protobuf;
211+
let on_columns = ctx.encode_children_expressions(&self.on_columns)?;
212+
Ok(Some(protobuf::PhysicalExprNode {
213+
expr_id: None,
214+
expr_type: Some(protobuf::physical_expr_node::ExprType::HashExpr(
215+
protobuf::PhysicalHashExprNode {
216+
on_columns,
217+
seed0: self.seed(),
218+
description: self.description.clone(),
219+
},
220+
)),
221+
}))
222+
}
223+
}
224+
225+
#[cfg(feature = "proto")]
226+
impl HashExpr {
227+
/// Reconstruct a [`HashExpr`] from its protobuf representation.
228+
///
229+
/// Takes the whole [`PhysicalExprNode`], the exact inverse of what
230+
/// [`PhysicalExpr::try_to_proto`] produces, so every expression's
231+
/// `try_from_proto` shares one signature. Child sub-expressions are
232+
/// decoded recursively via [`PhysicalExprDecodeCtx::decode`].
233+
///
234+
/// [`PhysicalExprNode`]: datafusion_proto_models::protobuf::PhysicalExprNode
235+
/// [`PhysicalExpr::try_to_proto`]: datafusion_physical_expr_common::physical_expr::PhysicalExpr::try_to_proto
236+
/// [`PhysicalExprDecodeCtx::decode`]: datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx::decode
237+
pub fn try_from_proto(
238+
node: &datafusion_proto_models::protobuf::PhysicalExprNode,
239+
ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,
240+
) -> Result<Arc<dyn PhysicalExpr>> {
241+
use datafusion_proto_models::protobuf;
242+
let hash_expr = match &node.expr_type {
243+
Some(protobuf::physical_expr_node::ExprType::HashExpr(h)) => h,
244+
_ => return internal_err!("PhysicalExprNode is not a HashExpr"),
245+
};
246+
let on_columns = ctx.decode_children_expressions(&hash_expr.on_columns)?;
247+
Ok(Arc::new(HashExpr::new(
248+
on_columns,
249+
SeededRandomState::with_seed(hash_expr.seed0),
250+
hash_expr.description.clone(),
251+
)))
252+
}
202253
}
203254

204255
/// Physical expression that checks join keys in a [`Map`] (hash table or array map).
@@ -498,6 +549,172 @@ mod tests {
498549
assert_eq!(compute_hash(&expr1), compute_hash(&expr2));
499550
}
500551

552+
#[cfg(feature = "proto")]
553+
mod proto_tests {
554+
use super::*;
555+
use arrow::datatypes::{DataType, Field};
556+
use datafusion_common::internal_datafusion_err;
557+
use datafusion_physical_expr_common::physical_expr::proto_decode::{
558+
PhysicalExprDecode, PhysicalExprDecodeCtx,
559+
};
560+
use datafusion_physical_expr_common::physical_expr::proto_encode::{
561+
PhysicalExprEncode, PhysicalExprEncodeCtx,
562+
};
563+
use datafusion_proto_models::protobuf;
564+
565+
struct TestEncoder;
566+
567+
impl PhysicalExprEncode for TestEncoder {
568+
fn encode(
569+
&self,
570+
expr: &Arc<dyn PhysicalExpr>,
571+
) -> Result<protobuf::PhysicalExprNode> {
572+
let ctx = PhysicalExprEncodeCtx::new(self);
573+
expr.try_to_proto(&ctx)?.ok_or_else(|| {
574+
internal_datafusion_err!("test encoder cannot encode {expr:?}")
575+
})
576+
}
577+
}
578+
579+
struct TestDecoder;
580+
581+
impl PhysicalExprDecode for TestDecoder {
582+
fn decode(
583+
&self,
584+
node: &protobuf::PhysicalExprNode,
585+
schema: &Schema,
586+
) -> Result<Arc<dyn PhysicalExpr>> {
587+
let ctx = PhysicalExprDecodeCtx::new(schema, self);
588+
match &node.expr_type {
589+
Some(protobuf::physical_expr_node::ExprType::Column(_)) => {
590+
Column::try_from_proto(node, &ctx)
591+
}
592+
_ => internal_err!("test decoder cannot decode {node:?}"),
593+
}
594+
}
595+
}
596+
597+
fn test_decode_ctx<'a>(
598+
schema: &'a Schema,
599+
decoder: &'a TestDecoder,
600+
) -> PhysicalExprDecodeCtx<'a> {
601+
PhysicalExprDecodeCtx::new(schema, decoder)
602+
}
603+
604+
#[test]
605+
fn hash_expr_try_to_proto() {
606+
let expr = HashExpr::new(
607+
vec![Arc::new(Column::new("a", 0)), Arc::new(Column::new("b", 1))],
608+
SeededRandomState::with_seed(42),
609+
"hash_join".to_string(),
610+
);
611+
let encoder = TestEncoder;
612+
let ctx = PhysicalExprEncodeCtx::new(&encoder);
613+
614+
let proto = expr.try_to_proto(&ctx).unwrap().unwrap();
615+
616+
assert_eq!(proto.expr_id, None);
617+
let hash_expr = match proto.expr_type.unwrap() {
618+
protobuf::physical_expr_node::ExprType::HashExpr(hash_expr) => hash_expr,
619+
other => panic!("expected HashExpr, got {other:?}"),
620+
};
621+
assert_eq!(hash_expr.seed0, 42);
622+
assert_eq!(hash_expr.description, "hash_join");
623+
assert_eq!(hash_expr.on_columns.len(), 2);
624+
assert!(
625+
hash_expr
626+
.on_columns
627+
.iter()
628+
.all(|expr| expr.expr_id.is_none())
629+
);
630+
}
631+
632+
#[test]
633+
fn hash_expr_try_from_proto() {
634+
let schema = Schema::new(vec![
635+
Field::new("a", DataType::Int32, false),
636+
Field::new("b", DataType::Utf8, true),
637+
]);
638+
let decoder = TestDecoder;
639+
let ctx = test_decode_ctx(&schema, &decoder);
640+
let proto = protobuf::PhysicalExprNode {
641+
expr_id: None,
642+
expr_type: Some(protobuf::physical_expr_node::ExprType::HashExpr(
643+
protobuf::PhysicalHashExprNode {
644+
on_columns: vec![
645+
protobuf::PhysicalExprNode {
646+
expr_id: None,
647+
expr_type: Some(
648+
protobuf::physical_expr_node::ExprType::Column(
649+
protobuf::PhysicalColumn {
650+
name: "a".to_string(),
651+
index: 0,
652+
},
653+
),
654+
),
655+
},
656+
protobuf::PhysicalExprNode {
657+
expr_id: None,
658+
expr_type: Some(
659+
protobuf::physical_expr_node::ExprType::Column(
660+
protobuf::PhysicalColumn {
661+
name: "b".to_string(),
662+
index: 1,
663+
},
664+
),
665+
),
666+
},
667+
],
668+
seed0: 42,
669+
description: "hash_join".to_string(),
670+
},
671+
)),
672+
};
673+
674+
let expr = HashExpr::try_from_proto(&proto, &ctx).unwrap();
675+
let expr = expr.downcast_ref::<HashExpr>().unwrap();
676+
677+
assert_eq!(expr.seed(), 42);
678+
assert_eq!(expr.description(), "hash_join");
679+
assert_eq!(expr.on_columns().len(), 2);
680+
assert_eq!(
681+
expr.on_columns()[0]
682+
.downcast_ref::<Column>()
683+
.map(|col| (col.name(), col.index())),
684+
Some(("a", 0))
685+
);
686+
assert_eq!(
687+
expr.on_columns()[1]
688+
.downcast_ref::<Column>()
689+
.map(|col| (col.name(), col.index())),
690+
Some(("b", 1))
691+
);
692+
}
693+
694+
#[test]
695+
fn hash_expr_try_from_proto_rejects_wrong_node_type() {
696+
let schema = Schema::empty();
697+
let decoder = TestDecoder;
698+
let ctx = test_decode_ctx(&schema, &decoder);
699+
let proto = protobuf::PhysicalExprNode {
700+
expr_id: None,
701+
expr_type: Some(protobuf::physical_expr_node::ExprType::Column(
702+
protobuf::PhysicalColumn {
703+
name: "a".to_string(),
704+
index: 0,
705+
},
706+
)),
707+
};
708+
709+
let err = HashExpr::try_from_proto(&proto, &ctx).unwrap_err();
710+
assert!(
711+
err.to_string()
712+
.contains("PhysicalExprNode is not a HashExpr"),
713+
"{err}"
714+
);
715+
}
716+
}
717+
501718
#[test]
502719
fn test_hash_table_lookup_expr_eq_same() {
503720
let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0));

‎datafusion/proto/src/physical_plan/from_proto.rs‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ use datafusion_physical_plan::expressions::{
4848
BinaryExpr, CaseExpr, CastExpr, Column, InListExpr, IsNotNullExpr, IsNullExpr,
4949
LikeExpr, Literal, NegativeExpr, NotExpr, TryCastExpr, UnKnownColumn,
5050
};
51-
use datafusion_physical_plan::joins::{HashExpr, SeededRandomState};
51+
use datafusion_physical_plan::joins::HashExpr;
5252
use datafusion_physical_plan::windows::{create_window_expr, schema_add_window_field};
5353
use datafusion_physical_plan::{
5454
Partitioning, PhysicalExpr, RangePartitioning, SplitPoint, WindowExpr,
@@ -412,19 +412,7 @@ pub fn parse_physical_expr_with_converter(
412412
)
413413
}
414414
ExprType::LikeExpr(_) => LikeExpr::try_from_proto(proto, &decode_ctx)?,
415-
ExprType::HashExpr(hash_expr) => {
416-
let on_columns = parse_physical_exprs(
417-
&hash_expr.on_columns,
418-
ctx,
419-
input_schema,
420-
proto_converter,
421-
)?;
422-
Arc::new(HashExpr::new(
423-
on_columns,
424-
SeededRandomState::with_seed(hash_expr.seed0),
425-
hash_expr.description.clone(),
426-
))
427-
}
415+
ExprType::HashExpr(_) => HashExpr::try_from_proto(proto, &decode_ctx)?,
428416
ExprType::ScalarSubquery(sq) => {
429417
let data_type: arrow::datatypes::DataType = sq
430418
.data_type

‎datafusion/proto/src/physical_plan/to_proto.rs‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@ use datafusion_physical_plan::expressions::{
3939
CaseExpr, CastExpr, DynamicFilterPhysicalExpr, IsNotNullExpr, IsNullExpr, Literal,
4040
NotExpr, TryCastExpr, UnKnownColumn,
4141
};
42-
use datafusion_physical_plan::joins::HashExpr;
4342
use datafusion_physical_plan::udaf::AggregateFunctionExpr;
4443
use datafusion_physical_plan::windows::{PlainAggregateWindowExpr, WindowUDFExpr};
4544
use datafusion_physical_plan::{
@@ -439,21 +438,6 @@ pub fn serialize_physical_expr_with_converter(
439438
},
440439
)),
441440
})
442-
} else if let Some(expr) = expr.downcast_ref::<HashExpr>() {
443-
Ok(protobuf::PhysicalExprNode {
444-
expr_id,
445-
expr_type: Some(protobuf::physical_expr_node::ExprType::HashExpr(
446-
protobuf::PhysicalHashExprNode {
447-
on_columns: serialize_physical_exprs(
448-
expr.on_columns(),
449-
codec,
450-
proto_converter,
451-
)?,
452-
seed0: expr.seed(),
453-
description: expr.description().to_string(),
454-
},
455-
)),
456-
})
457441
} else if let Some(expr) = expr.downcast_ref::<ScalarSubqueryExpr>() {
458442
Ok(protobuf::PhysicalExprNode {
459443
expr_id,

0 commit comments

Comments
 (0)