Skip to content

Commit 8a6e069

Browse files
committed
refactor(proto): migrate ExplainExec serde
Move ExplainExec serialization into its execution plan implementation. This removes its live central dispatch while preserving the existing wire format and deprecated compatibility entry points. Add roundtrip coverage for every StringifiedPlan variant and all stored ExplainExec fields. Refs apache#23511 Signed-off-by: Jiawei Zhao <[email protected]>
1 parent 9a50c67 commit 8a6e069

3 files changed

Lines changed: 281 additions & 32 deletions

File tree

‎datafusion/physical-plan/src/explain.rs‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,188 @@ impl ExecutionPlan for ExplainExec {
185185
futures::stream::iter(vec![Ok(record_batch)]),
186186
)))
187187
}
188+
189+
#[cfg(feature = "proto")]
190+
fn try_to_proto(
191+
&self,
192+
_ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
193+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
194+
use datafusion_proto_models::protobuf;
195+
196+
Ok(Some(protobuf::PhysicalPlanNode {
197+
physical_plan_type: Some(
198+
protobuf::physical_plan_node::PhysicalPlanType::Explain(
199+
protobuf::ExplainExecNode {
200+
schema: Some(self.schema().as_ref().try_into()?),
201+
stringified_plans: self
202+
.stringified_plans()
203+
.iter()
204+
.map(stringified_plan_to_proto)
205+
.collect(),
206+
verbose: self.verbose(),
207+
},
208+
),
209+
),
210+
}))
211+
}
212+
}
213+
214+
#[cfg(feature = "proto")]
215+
impl ExplainExec {
216+
/// Reconstruct an [`ExplainExec`] from its protobuf representation.
217+
pub fn try_from_proto(
218+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
219+
_ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
220+
) -> Result<Arc<dyn ExecutionPlan>> {
221+
use datafusion_proto_models::protobuf;
222+
223+
let explain = crate::expect_plan_variant!(
224+
node,
225+
protobuf::physical_plan_node::PhysicalPlanType::Explain,
226+
"ExplainExec",
227+
);
228+
let schema = explain.schema.as_ref().ok_or_else(|| {
229+
datafusion_common::internal_datafusion_err!(
230+
"ExplainExec is missing required field 'schema'"
231+
)
232+
})?;
233+
Ok(Arc::new(ExplainExec::new(
234+
Arc::new(arrow::datatypes::Schema::try_from(schema)?),
235+
explain
236+
.stringified_plans
237+
.iter()
238+
.map(stringified_plan_from_proto)
239+
.collect(),
240+
explain.verbose,
241+
)))
242+
}
243+
}
244+
245+
#[cfg(feature = "proto")]
246+
fn stringified_plan_to_proto(
247+
stringified_plan: &StringifiedPlan,
248+
) -> datafusion_proto_models::protobuf::StringifiedPlan {
249+
use datafusion_common::display::PlanType;
250+
use datafusion_proto_models::datafusion_common::EmptyMessage;
251+
use datafusion_proto_models::protobuf;
252+
use protobuf::plan_type::PlanTypeEnum::{
253+
AnalyzedLogicalPlan, FinalAnalyzedLogicalPlan, FinalLogicalPlan,
254+
FinalPhysicalPlan, FinalPhysicalPlanWithSchema, FinalPhysicalPlanWithStats,
255+
InitialLogicalPlan, InitialPhysicalPlan, InitialPhysicalPlanWithSchema,
256+
InitialPhysicalPlanWithStats, OptimizedLogicalPlan, OptimizedPhysicalPlan,
257+
PhysicalPlanError,
258+
};
259+
260+
protobuf::StringifiedPlan {
261+
plan_type: match stringified_plan.clone().plan_type {
262+
PlanType::InitialLogicalPlan => Some(protobuf::PlanType {
263+
plan_type_enum: Some(InitialLogicalPlan(EmptyMessage {})),
264+
}),
265+
PlanType::AnalyzedLogicalPlan { analyzer_name } => Some(protobuf::PlanType {
266+
plan_type_enum: Some(AnalyzedLogicalPlan(
267+
protobuf::AnalyzedLogicalPlanType { analyzer_name },
268+
)),
269+
}),
270+
PlanType::FinalAnalyzedLogicalPlan => Some(protobuf::PlanType {
271+
plan_type_enum: Some(FinalAnalyzedLogicalPlan(EmptyMessage {})),
272+
}),
273+
PlanType::OptimizedLogicalPlan { optimizer_name } => {
274+
Some(protobuf::PlanType {
275+
plan_type_enum: Some(OptimizedLogicalPlan(
276+
protobuf::OptimizedLogicalPlanType { optimizer_name },
277+
)),
278+
})
279+
}
280+
PlanType::FinalLogicalPlan => Some(protobuf::PlanType {
281+
plan_type_enum: Some(FinalLogicalPlan(EmptyMessage {})),
282+
}),
283+
PlanType::InitialPhysicalPlan => Some(protobuf::PlanType {
284+
plan_type_enum: Some(InitialPhysicalPlan(EmptyMessage {})),
285+
}),
286+
PlanType::OptimizedPhysicalPlan { optimizer_name } => {
287+
Some(protobuf::PlanType {
288+
plan_type_enum: Some(OptimizedPhysicalPlan(
289+
protobuf::OptimizedPhysicalPlanType { optimizer_name },
290+
)),
291+
})
292+
}
293+
PlanType::FinalPhysicalPlan => Some(protobuf::PlanType {
294+
plan_type_enum: Some(FinalPhysicalPlan(EmptyMessage {})),
295+
}),
296+
PlanType::InitialPhysicalPlanWithStats => Some(protobuf::PlanType {
297+
plan_type_enum: Some(InitialPhysicalPlanWithStats(EmptyMessage {})),
298+
}),
299+
PlanType::InitialPhysicalPlanWithSchema => Some(protobuf::PlanType {
300+
plan_type_enum: Some(InitialPhysicalPlanWithSchema(EmptyMessage {})),
301+
}),
302+
PlanType::FinalPhysicalPlanWithStats => Some(protobuf::PlanType {
303+
plan_type_enum: Some(FinalPhysicalPlanWithStats(EmptyMessage {})),
304+
}),
305+
PlanType::FinalPhysicalPlanWithSchema => Some(protobuf::PlanType {
306+
plan_type_enum: Some(FinalPhysicalPlanWithSchema(EmptyMessage {})),
307+
}),
308+
PlanType::PhysicalPlanError => Some(protobuf::PlanType {
309+
plan_type_enum: Some(PhysicalPlanError(EmptyMessage {})),
310+
}),
311+
},
312+
plan: stringified_plan.plan.to_string(),
313+
}
314+
}
315+
316+
#[cfg(feature = "proto")]
317+
fn stringified_plan_from_proto(
318+
stringified_plan: &datafusion_proto_models::protobuf::StringifiedPlan,
319+
) -> StringifiedPlan {
320+
use datafusion_common::display::PlanType;
321+
use datafusion_proto_models::protobuf::plan_type::PlanTypeEnum::{
322+
AnalyzedLogicalPlan, FinalAnalyzedLogicalPlan, FinalLogicalPlan,
323+
FinalPhysicalPlan, FinalPhysicalPlanWithSchema, FinalPhysicalPlanWithStats,
324+
InitialLogicalPlan, InitialPhysicalPlan, InitialPhysicalPlanWithSchema,
325+
InitialPhysicalPlanWithStats, OptimizedLogicalPlan, OptimizedPhysicalPlan,
326+
PhysicalPlanError,
327+
};
328+
use datafusion_proto_models::protobuf::{
329+
AnalyzedLogicalPlanType, OptimizedLogicalPlanType, OptimizedPhysicalPlanType,
330+
};
331+
332+
StringifiedPlan {
333+
plan_type: match stringified_plan
334+
.plan_type
335+
.as_ref()
336+
.and_then(|plan_type| plan_type.plan_type_enum.as_ref())
337+
.unwrap_or_else(|| {
338+
panic!(
339+
"Cannot create protobuf::StringifiedPlan from {stringified_plan:?}"
340+
)
341+
}) {
342+
InitialLogicalPlan(_) => PlanType::InitialLogicalPlan,
343+
AnalyzedLogicalPlan(AnalyzedLogicalPlanType { analyzer_name }) => {
344+
PlanType::AnalyzedLogicalPlan {
345+
analyzer_name: analyzer_name.clone(),
346+
}
347+
}
348+
FinalAnalyzedLogicalPlan(_) => PlanType::FinalAnalyzedLogicalPlan,
349+
OptimizedLogicalPlan(OptimizedLogicalPlanType { optimizer_name }) => {
350+
PlanType::OptimizedLogicalPlan {
351+
optimizer_name: optimizer_name.clone(),
352+
}
353+
}
354+
FinalLogicalPlan(_) => PlanType::FinalLogicalPlan,
355+
InitialPhysicalPlan(_) => PlanType::InitialPhysicalPlan,
356+
InitialPhysicalPlanWithStats(_) => PlanType::InitialPhysicalPlanWithStats,
357+
InitialPhysicalPlanWithSchema(_) => PlanType::InitialPhysicalPlanWithSchema,
358+
OptimizedPhysicalPlan(OptimizedPhysicalPlanType { optimizer_name }) => {
359+
PlanType::OptimizedPhysicalPlan {
360+
optimizer_name: optimizer_name.clone(),
361+
}
362+
}
363+
FinalPhysicalPlan(_) => PlanType::FinalPhysicalPlan,
364+
FinalPhysicalPlanWithStats(_) => PlanType::FinalPhysicalPlanWithStats,
365+
FinalPhysicalPlanWithSchema(_) => PlanType::FinalPhysicalPlanWithSchema,
366+
PhysicalPlanError(_) => PlanType::PhysicalPlanError,
367+
},
368+
plan: Arc::new(stringified_plan.plan.clone()),
369+
}
188370
}
189371

190372
/// If this plan should be shown, given the previous plan that was

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 28 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ use arrow::compute::SortOptions;
2525
use arrow::datatypes::{IntervalMonthDayNanoType, Schema, SchemaRef};
2626
use datafusion_catalog::memory::MemorySourceConfig;
2727
use datafusion_common::config::CsvOptions;
28-
use datafusion_common::display::StringifiedPlan;
2928
use datafusion_common::format::ExplainFormat;
3029
use datafusion_common::{
3130
DataFusionError, JoinType, NullEquality, Result, internal_datafusion_err,
@@ -753,8 +752,8 @@ pub trait PhysicalPlanNodeExt: Sized {
753752
};
754753
let decode_ctx = ExecutionPlanDecodeCtx::new(&plan_decoder);
755754
match plan {
756-
PhysicalPlanType::Explain(explain) => {
757-
self.try_into_explain_physical_plan(explain, ctx, proto_converter)
755+
PhysicalPlanType::Explain(_) => {
756+
ExplainExec::try_from_proto(self.node(), &decode_ctx)
758757
}
759758
PhysicalPlanType::Projection(_) => {
760759
ProjectionExec::try_from_proto(self.node(), &decode_ctx)
@@ -905,10 +904,6 @@ pub trait PhysicalPlanNodeExt: Sized {
905904
return Ok(node);
906905
}
907906

908-
if let Some(exec) = plan.downcast_ref::<ExplainExec>() {
909-
return protobuf::PhysicalPlanNode::try_from_explain_exec(exec, codec);
910-
}
911-
912907
if let Some(exec) = plan.downcast_ref::<AnalyzeExec>() {
913908
return protobuf::PhysicalPlanNode::try_from_analyze_exec(
914909
exec,
@@ -1118,21 +1113,22 @@ pub trait PhysicalPlanNodeExt: Sized {
11181113
}
11191114
}
11201115

1116+
#[deprecated(
1117+
since = "55.0.0",
1118+
note = "unused by DataFusion; `ExplainExec` deserializes itself via `ExplainExec::try_from_proto`"
1119+
)]
11211120
fn try_into_explain_physical_plan(
11221121
&self,
1123-
explain: &protobuf::ExplainExecNode,
1124-
_ctx: &PhysicalPlanDecodeContext<'_>,
1125-
_proto_converter: &dyn PhysicalProtoConverterExtension,
1122+
_explain: &protobuf::ExplainExecNode,
1123+
ctx: &PhysicalPlanDecodeContext<'_>,
1124+
proto_converter: &dyn PhysicalProtoConverterExtension,
11261125
) -> Result<Arc<dyn ExecutionPlan>> {
1127-
Ok(Arc::new(ExplainExec::new(
1128-
Arc::new(explain.schema.as_ref().unwrap().try_into()?),
1129-
explain
1130-
.stringified_plans
1131-
.iter()
1132-
.map(StringifiedPlan::from_proto)
1133-
.collect(),
1134-
explain.verbose,
1135-
)))
1126+
let plan_decoder = ConverterPlanDecoder {
1127+
ctx,
1128+
proto_converter,
1129+
};
1130+
let decode_ctx = ExecutionPlanDecodeCtx::new(&plan_decoder);
1131+
ExplainExec::try_from_proto(self.node(), &decode_ctx)
11361132
}
11371133

11381134
#[deprecated(
@@ -2749,22 +2745,22 @@ pub trait PhysicalPlanNodeExt: Sized {
27492745
)))
27502746
}
27512747

2748+
#[deprecated(
2749+
since = "55.0.0",
2750+
note = "unused by DataFusion; `ExplainExec` serializes itself via `ExecutionPlan::try_to_proto`"
2751+
)]
27522752
fn try_from_explain_exec(
27532753
exec: &ExplainExec,
2754-
_codec: &dyn PhysicalExtensionCodec,
2754+
codec: &dyn PhysicalExtensionCodec,
27552755
) -> Result<protobuf::PhysicalPlanNode> {
2756-
Ok(protobuf::PhysicalPlanNode {
2757-
physical_plan_type: Some(PhysicalPlanType::Explain(
2758-
protobuf::ExplainExecNode {
2759-
schema: Some(exec.schema().as_ref().try_into()?),
2760-
stringified_plans: exec
2761-
.stringified_plans()
2762-
.iter()
2763-
.map(protobuf::StringifiedPlan::from_proto)
2764-
.collect(),
2765-
verbose: exec.verbose(),
2766-
},
2767-
)),
2756+
let proto_converter = DefaultPhysicalProtoConverter {};
2757+
let plan_encoder = ConverterPlanEncoder {
2758+
codec,
2759+
proto_converter: &proto_converter,
2760+
};
2761+
let encode_ctx = ExecutionPlanEncodeCtx::new(&plan_encoder);
2762+
exec.try_to_proto(&encode_ctx)?.ok_or_else(|| {
2763+
internal_datafusion_err!("ExplainExec did not serialize itself")
27682764
})
27692765
}
27702766

‎datafusion/proto/tests/cases/roundtrip_physical_plan.rs‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;
6666
use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
6767
use datafusion::physical_plan::coop::CooperativeExec;
6868
use datafusion::physical_plan::empty::EmptyExec;
69+
use datafusion::physical_plan::explain::ExplainExec;
6970
use datafusion::physical_plan::expressions::{
7071
BinaryExpr, Column, DynamicFilterPhysicalExpr, NotExpr, PhysicalSortExpr, binary,
7172
cast, col, in_list, like, lit,
@@ -97,6 +98,7 @@ use datafusion::physical_plan::{
9798
use datafusion::prelude::{ParquetReadOptions, SessionContext};
9899
use datafusion::scalar::ScalarValue;
99100
use datafusion_common::config::{ConfigOptions, TableParquetOptions};
101+
use datafusion_common::display::{PlanType, StringifiedPlan};
100102
use datafusion_common::file_options::csv_writer::CsvWriterOptions;
101103
use datafusion_common::file_options::json_writer::JsonWriterOptions;
102104
use datafusion_common::parsers::CompressionTypeVariant;
@@ -1956,6 +1958,75 @@ fn roundtrip_analyze() -> Result<()> {
19561958
))
19571959
}
19581960

1961+
#[test]
1962+
fn roundtrip_explain() -> Result<()> {
1963+
let schema = Arc::new(Schema::new(vec![
1964+
Field::new("plan_type", DataType::Utf8, false),
1965+
Field::new("plan", DataType::Utf8, false),
1966+
]));
1967+
let stringified_plans = vec![
1968+
StringifiedPlan::new(PlanType::InitialLogicalPlan, "initial logical"),
1969+
StringifiedPlan::new(
1970+
PlanType::AnalyzedLogicalPlan {
1971+
analyzer_name: "analyzer".to_string(),
1972+
},
1973+
"analyzed logical",
1974+
),
1975+
StringifiedPlan::new(PlanType::FinalAnalyzedLogicalPlan, "final analyzed"),
1976+
StringifiedPlan::new(
1977+
PlanType::OptimizedLogicalPlan {
1978+
optimizer_name: "logical optimizer".to_string(),
1979+
},
1980+
"optimized logical",
1981+
),
1982+
StringifiedPlan::new(PlanType::FinalLogicalPlan, "final logical"),
1983+
StringifiedPlan::new(PlanType::InitialPhysicalPlan, "initial physical"),
1984+
StringifiedPlan::new(
1985+
PlanType::InitialPhysicalPlanWithStats,
1986+
"initial physical with stats",
1987+
),
1988+
StringifiedPlan::new(
1989+
PlanType::InitialPhysicalPlanWithSchema,
1990+
"initial physical with schema",
1991+
),
1992+
StringifiedPlan::new(
1993+
PlanType::OptimizedPhysicalPlan {
1994+
optimizer_name: "physical optimizer".to_string(),
1995+
},
1996+
"optimized physical",
1997+
),
1998+
StringifiedPlan::new(PlanType::FinalPhysicalPlan, "final physical"),
1999+
StringifiedPlan::new(
2000+
PlanType::FinalPhysicalPlanWithStats,
2001+
"final physical with stats",
2002+
),
2003+
StringifiedPlan::new(
2004+
PlanType::FinalPhysicalPlanWithSchema,
2005+
"final physical with schema",
2006+
),
2007+
StringifiedPlan::new(PlanType::PhysicalPlanError, "physical plan error"),
2008+
];
2009+
let explain = Arc::new(ExplainExec::new(
2010+
Arc::clone(&schema),
2011+
stringified_plans.clone(),
2012+
true,
2013+
));
2014+
2015+
let ctx = SessionContext::new();
2016+
let roundtripped = roundtrip_test_and_return(
2017+
explain,
2018+
&ctx,
2019+
&DefaultPhysicalExtensionCodec {},
2020+
&DefaultPhysicalProtoConverter {},
2021+
)?;
2022+
let roundtripped = roundtripped.downcast_ref::<ExplainExec>().unwrap();
2023+
2024+
assert_eq!(roundtripped.schema(), schema);
2025+
assert_eq!(roundtripped.stringified_plans(), stringified_plans);
2026+
assert!(roundtripped.verbose());
2027+
Ok(())
2028+
}
2029+
19592030
#[tokio::test]
19602031
async fn roundtrip_json_source() -> Result<()> {
19612032
let ctx = SessionContext::new();

0 commit comments

Comments
 (0)