Skip to content

Commit e119229

Browse files
committed
refactor(proto): migrate ExplainExec serde
Move ExplainExec serialization into its execution plan implementation. This removes its live central dispatch while preserving the existing wire format and deprecated compatibility entry points. Add roundtrip coverage for every StringifiedPlan variant and all stored ExplainExec fields. Refs apache#23511 Signed-off-by: Jiawei Zhao <[email protected]>
1 parent 1763417 commit e119229

3 files changed

Lines changed: 281 additions & 32 deletions

File tree

‎datafusion/physical-plan/src/explain.rs‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,188 @@ impl ExecutionPlan for ExplainExec {
185185
futures::stream::iter(vec![Ok(record_batch)]),
186186
)))
187187
}
188+
189+
#[cfg(feature = "proto")]
190+
fn try_to_proto(
191+
&self,
192+
_ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
193+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
194+
use datafusion_proto_models::protobuf;
195+
196+
Ok(Some(protobuf::PhysicalPlanNode {
197+
physical_plan_type: Some(
198+
protobuf::physical_plan_node::PhysicalPlanType::Explain(
199+
protobuf::ExplainExecNode {
200+
schema: Some(self.schema().as_ref().try_into()?),
201+
stringified_plans: self
202+
.stringified_plans()
203+
.iter()
204+
.map(stringified_plan_to_proto)
205+
.collect(),
206+
verbose: self.verbose(),
207+
},
208+
),
209+
),
210+
}))
211+
}
212+
}
213+
214+
#[cfg(feature = "proto")]
215+
impl ExplainExec {
216+
/// Reconstruct an [`ExplainExec`] from its protobuf representation.
217+
pub fn try_from_proto(
218+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
219+
_ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
220+
) -> Result<Arc<dyn ExecutionPlan>> {
221+
use datafusion_proto_models::protobuf;
222+
223+
let explain = crate::expect_plan_variant!(
224+
node,
225+
protobuf::physical_plan_node::PhysicalPlanType::Explain,
226+
"ExplainExec",
227+
);
228+
let schema = explain.schema.as_ref().ok_or_else(|| {
229+
datafusion_common::internal_datafusion_err!(
230+
"ExplainExec is missing required field 'schema'"
231+
)
232+
})?;
233+
Ok(Arc::new(ExplainExec::new(
234+
Arc::new(arrow::datatypes::Schema::try_from(schema)?),
235+
explain
236+
.stringified_plans
237+
.iter()
238+
.map(stringified_plan_from_proto)
239+
.collect(),
240+
explain.verbose,
241+
)))
242+
}
243+
}
244+
245+
#[cfg(feature = "proto")]
246+
fn stringified_plan_to_proto(
247+
stringified_plan: &StringifiedPlan,
248+
) -> datafusion_proto_models::protobuf::StringifiedPlan {
249+
use datafusion_common::display::PlanType;
250+
use datafusion_proto_models::datafusion_common::EmptyMessage;
251+
use datafusion_proto_models::protobuf;
252+
use protobuf::plan_type::PlanTypeEnum::{
253+
AnalyzedLogicalPlan, FinalAnalyzedLogicalPlan, FinalLogicalPlan,
254+
FinalPhysicalPlan, FinalPhysicalPlanWithSchema, FinalPhysicalPlanWithStats,
255+
InitialLogicalPlan, InitialPhysicalPlan, InitialPhysicalPlanWithSchema,
256+
InitialPhysicalPlanWithStats, OptimizedLogicalPlan, OptimizedPhysicalPlan,
257+
PhysicalPlanError,
258+
};
259+
260+
protobuf::StringifiedPlan {
261+
plan_type: match stringified_plan.clone().plan_type {
262+
PlanType::InitialLogicalPlan => Some(protobuf::PlanType {
263+
plan_type_enum: Some(InitialLogicalPlan(EmptyMessage {})),
264+
}),
265+
PlanType::AnalyzedLogicalPlan { analyzer_name } => Some(protobuf::PlanType {
266+
plan_type_enum: Some(AnalyzedLogicalPlan(
267+
protobuf::AnalyzedLogicalPlanType { analyzer_name },
268+
)),
269+
}),
270+
PlanType::FinalAnalyzedLogicalPlan => Some(protobuf::PlanType {
271+
plan_type_enum: Some(FinalAnalyzedLogicalPlan(EmptyMessage {})),
272+
}),
273+
PlanType::OptimizedLogicalPlan { optimizer_name } => {
274+
Some(protobuf::PlanType {
275+
plan_type_enum: Some(OptimizedLogicalPlan(
276+
protobuf::OptimizedLogicalPlanType { optimizer_name },
277+
)),
278+
})
279+
}
280+
PlanType::FinalLogicalPlan => Some(protobuf::PlanType {
281+
plan_type_enum: Some(FinalLogicalPlan(EmptyMessage {})),
282+
}),
283+
PlanType::InitialPhysicalPlan => Some(protobuf::PlanType {
284+
plan_type_enum: Some(InitialPhysicalPlan(EmptyMessage {})),
285+
}),
286+
PlanType::OptimizedPhysicalPlan { optimizer_name } => {
287+
Some(protobuf::PlanType {
288+
plan_type_enum: Some(OptimizedPhysicalPlan(
289+
protobuf::OptimizedPhysicalPlanType { optimizer_name },
290+
)),
291+
})
292+
}
293+
PlanType::FinalPhysicalPlan => Some(protobuf::PlanType {
294+
plan_type_enum: Some(FinalPhysicalPlan(EmptyMessage {})),
295+
}),
296+
PlanType::InitialPhysicalPlanWithStats => Some(protobuf::PlanType {
297+
plan_type_enum: Some(InitialPhysicalPlanWithStats(EmptyMessage {})),
298+
}),
299+
PlanType::InitialPhysicalPlanWithSchema => Some(protobuf::PlanType {
300+
plan_type_enum: Some(InitialPhysicalPlanWithSchema(EmptyMessage {})),
301+
}),
302+
PlanType::FinalPhysicalPlanWithStats => Some(protobuf::PlanType {
303+
plan_type_enum: Some(FinalPhysicalPlanWithStats(EmptyMessage {})),
304+
}),
305+
PlanType::FinalPhysicalPlanWithSchema => Some(protobuf::PlanType {
306+
plan_type_enum: Some(FinalPhysicalPlanWithSchema(EmptyMessage {})),
307+
}),
308+
PlanType::PhysicalPlanError => Some(protobuf::PlanType {
309+
plan_type_enum: Some(PhysicalPlanError(EmptyMessage {})),
310+
}),
311+
},
312+
plan: stringified_plan.plan.to_string(),
313+
}
314+
}
315+
316+
#[cfg(feature = "proto")]
317+
fn stringified_plan_from_proto(
318+
stringified_plan: &datafusion_proto_models::protobuf::StringifiedPlan,
319+
) -> StringifiedPlan {
320+
use datafusion_common::display::PlanType;
321+
use datafusion_proto_models::protobuf::plan_type::PlanTypeEnum::{
322+
AnalyzedLogicalPlan, FinalAnalyzedLogicalPlan, FinalLogicalPlan,
323+
FinalPhysicalPlan, FinalPhysicalPlanWithSchema, FinalPhysicalPlanWithStats,
324+
InitialLogicalPlan, InitialPhysicalPlan, InitialPhysicalPlanWithSchema,
325+
InitialPhysicalPlanWithStats, OptimizedLogicalPlan, OptimizedPhysicalPlan,
326+
PhysicalPlanError,
327+
};
328+
use datafusion_proto_models::protobuf::{
329+
AnalyzedLogicalPlanType, OptimizedLogicalPlanType, OptimizedPhysicalPlanType,
330+
};
331+
332+
StringifiedPlan {
333+
plan_type: match stringified_plan
334+
.plan_type
335+
.as_ref()
336+
.and_then(|plan_type| plan_type.plan_type_enum.as_ref())
337+
.unwrap_or_else(|| {
338+
panic!(
339+
"Cannot create protobuf::StringifiedPlan from {stringified_plan:?}"
340+
)
341+
}) {
342+
InitialLogicalPlan(_) => PlanType::InitialLogicalPlan,
343+
AnalyzedLogicalPlan(AnalyzedLogicalPlanType { analyzer_name }) => {
344+
PlanType::AnalyzedLogicalPlan {
345+
analyzer_name: analyzer_name.clone(),
346+
}
347+
}
348+
FinalAnalyzedLogicalPlan(_) => PlanType::FinalAnalyzedLogicalPlan,
349+
OptimizedLogicalPlan(OptimizedLogicalPlanType { optimizer_name }) => {
350+
PlanType::OptimizedLogicalPlan {
351+
optimizer_name: optimizer_name.clone(),
352+
}
353+
}
354+
FinalLogicalPlan(_) => PlanType::FinalLogicalPlan,
355+
InitialPhysicalPlan(_) => PlanType::InitialPhysicalPlan,
356+
InitialPhysicalPlanWithStats(_) => PlanType::InitialPhysicalPlanWithStats,
357+
InitialPhysicalPlanWithSchema(_) => PlanType::InitialPhysicalPlanWithSchema,
358+
OptimizedPhysicalPlan(OptimizedPhysicalPlanType { optimizer_name }) => {
359+
PlanType::OptimizedPhysicalPlan {
360+
optimizer_name: optimizer_name.clone(),
361+
}
362+
}
363+
FinalPhysicalPlan(_) => PlanType::FinalPhysicalPlan,
364+
FinalPhysicalPlanWithStats(_) => PlanType::FinalPhysicalPlanWithStats,
365+
FinalPhysicalPlanWithSchema(_) => PlanType::FinalPhysicalPlanWithSchema,
366+
PhysicalPlanError(_) => PlanType::PhysicalPlanError,
367+
},
368+
plan: Arc::new(stringified_plan.plan.clone()),
369+
}
188370
}
189371

190372
/// If this plan should be shown, given the previous plan that was

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 28 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ use std::sync::Arc;
2424
use arrow::datatypes::{IntervalMonthDayNanoType, Schema, SchemaRef};
2525
use datafusion_catalog::memory::MemorySourceConfig;
2626
use datafusion_common::config::CsvOptions;
27-
use datafusion_common::display::StringifiedPlan;
2827
use datafusion_common::format::ExplainFormat;
2928
use datafusion_common::{
3029
DataFusionError, JoinType, NullEquality, Result, internal_datafusion_err,
@@ -747,8 +746,8 @@ pub trait PhysicalPlanNodeExt: Sized {
747746
};
748747
let decode_ctx = ExecutionPlanDecodeCtx::new(&plan_decoder);
749748
match plan {
750-
PhysicalPlanType::Explain(explain) => {
751-
self.try_into_explain_physical_plan(explain, ctx, proto_converter)
749+
PhysicalPlanType::Explain(_) => {
750+
ExplainExec::try_from_proto(self.node(), &decode_ctx)
752751
}
753752
PhysicalPlanType::Projection(_) => {
754753
ProjectionExec::try_from_proto(self.node(), &decode_ctx)
@@ -900,10 +899,6 @@ pub trait PhysicalPlanNodeExt: Sized {
900899
return Ok(node);
901900
}
902901

903-
if let Some(exec) = plan.downcast_ref::<ExplainExec>() {
904-
return protobuf::PhysicalPlanNode::try_from_explain_exec(exec, codec);
905-
}
906-
907902
if let Some(exec) = plan.downcast_ref::<AnalyzeExec>() {
908903
return protobuf::PhysicalPlanNode::try_from_analyze_exec(
909904
exec,
@@ -1033,21 +1028,22 @@ pub trait PhysicalPlanNodeExt: Sized {
10331028
}
10341029
}
10351030

1031+
#[deprecated(
1032+
since = "55.0.0",
1033+
note = "unused by DataFusion; `ExplainExec` deserializes itself via `ExplainExec::try_from_proto`"
1034+
)]
10361035
fn try_into_explain_physical_plan(
10371036
&self,
1038-
explain: &protobuf::ExplainExecNode,
1039-
_ctx: &PhysicalPlanDecodeContext<'_>,
1040-
_proto_converter: &dyn PhysicalProtoConverterExtension,
1037+
_explain: &protobuf::ExplainExecNode,
1038+
ctx: &PhysicalPlanDecodeContext<'_>,
1039+
proto_converter: &dyn PhysicalProtoConverterExtension,
10411040
) -> Result<Arc<dyn ExecutionPlan>> {
1042-
Ok(Arc::new(ExplainExec::new(
1043-
Arc::new(explain.schema.as_ref().unwrap().try_into()?),
1044-
explain
1045-
.stringified_plans
1046-
.iter()
1047-
.map(StringifiedPlan::from_proto)
1048-
.collect(),
1049-
explain.verbose,
1050-
)))
1041+
let plan_decoder = ConverterPlanDecoder {
1042+
ctx,
1043+
proto_converter,
1044+
};
1045+
let decode_ctx = ExecutionPlanDecodeCtx::new(&plan_decoder);
1046+
ExplainExec::try_from_proto(self.node(), &decode_ctx)
10511047
}
10521048

10531049
#[deprecated(
@@ -2562,22 +2558,22 @@ pub trait PhysicalPlanNodeExt: Sized {
25622558
)))
25632559
}
25642560

2561+
#[deprecated(
2562+
since = "55.0.0",
2563+
note = "unused by DataFusion; `ExplainExec` serializes itself via `ExecutionPlan::try_to_proto`"
2564+
)]
25652565
fn try_from_explain_exec(
25662566
exec: &ExplainExec,
2567-
_codec: &dyn PhysicalExtensionCodec,
2567+
codec: &dyn PhysicalExtensionCodec,
25682568
) -> Result<protobuf::PhysicalPlanNode> {
2569-
Ok(protobuf::PhysicalPlanNode {
2570-
physical_plan_type: Some(PhysicalPlanType::Explain(
2571-
protobuf::ExplainExecNode {
2572-
schema: Some(exec.schema().as_ref().try_into()?),
2573-
stringified_plans: exec
2574-
.stringified_plans()
2575-
.iter()
2576-
.map(protobuf::StringifiedPlan::from_proto)
2577-
.collect(),
2578-
verbose: exec.verbose(),
2579-
},
2580-
)),
2569+
let proto_converter = DefaultPhysicalProtoConverter {};
2570+
let plan_encoder = ConverterPlanEncoder {
2571+
codec,
2572+
proto_converter: &proto_converter,
2573+
};
2574+
let encode_ctx = ExecutionPlanEncodeCtx::new(&plan_encoder);
2575+
exec.try_to_proto(&encode_ctx)?.ok_or_else(|| {
2576+
internal_datafusion_err!("ExplainExec did not serialize itself")
25812577
})
25822578
}
25832579

‎datafusion/proto/tests/cases/roundtrip_physical_plan.rs‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;
6666
use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
6767
use datafusion::physical_plan::coop::CooperativeExec;
6868
use datafusion::physical_plan::empty::EmptyExec;
69+
use datafusion::physical_plan::explain::ExplainExec;
6970
use datafusion::physical_plan::expressions::{
7071
BinaryExpr, Column, DynamicFilterPhysicalExpr, NotExpr, PhysicalSortExpr, binary,
7172
cast, col, in_list, like, lit,
@@ -97,6 +98,7 @@ use datafusion::physical_plan::{
9798
use datafusion::prelude::{ParquetReadOptions, SessionContext};
9899
use datafusion::scalar::ScalarValue;
99100
use datafusion_common::config::{ConfigOptions, TableParquetOptions};
101+
use datafusion_common::display::{PlanType, StringifiedPlan};
100102
use datafusion_common::file_options::csv_writer::CsvWriterOptions;
101103
use datafusion_common::file_options::json_writer::JsonWriterOptions;
102104
use datafusion_common::parsers::CompressionTypeVariant;
@@ -1956,6 +1958,75 @@ fn roundtrip_analyze() -> Result<()> {
19561958
))
19571959
}
19581960

1961+
#[test]
1962+
fn roundtrip_explain() -> Result<()> {
1963+
let schema = Arc::new(Schema::new(vec![
1964+
Field::new("plan_type", DataType::Utf8, false),
1965+
Field::new("plan", DataType::Utf8, false),
1966+
]));
1967+
let stringified_plans = vec![
1968+
StringifiedPlan::new(PlanType::InitialLogicalPlan, "initial logical"),
1969+
StringifiedPlan::new(
1970+
PlanType::AnalyzedLogicalPlan {
1971+
analyzer_name: "analyzer".to_string(),
1972+
},
1973+
"analyzed logical",
1974+
),
1975+
StringifiedPlan::new(PlanType::FinalAnalyzedLogicalPlan, "final analyzed"),
1976+
StringifiedPlan::new(
1977+
PlanType::OptimizedLogicalPlan {
1978+
optimizer_name: "logical optimizer".to_string(),
1979+
},
1980+
"optimized logical",
1981+
),
1982+
StringifiedPlan::new(PlanType::FinalLogicalPlan, "final logical"),
1983+
StringifiedPlan::new(PlanType::InitialPhysicalPlan, "initial physical"),
1984+
StringifiedPlan::new(
1985+
PlanType::InitialPhysicalPlanWithStats,
1986+
"initial physical with stats",
1987+
),
1988+
StringifiedPlan::new(
1989+
PlanType::InitialPhysicalPlanWithSchema,
1990+
"initial physical with schema",
1991+
),
1992+
StringifiedPlan::new(
1993+
PlanType::OptimizedPhysicalPlan {
1994+
optimizer_name: "physical optimizer".to_string(),
1995+
},
1996+
"optimized physical",
1997+
),
1998+
StringifiedPlan::new(PlanType::FinalPhysicalPlan, "final physical"),
1999+
StringifiedPlan::new(
2000+
PlanType::FinalPhysicalPlanWithStats,
2001+
"final physical with stats",
2002+
),
2003+
StringifiedPlan::new(
2004+
PlanType::FinalPhysicalPlanWithSchema,
2005+
"final physical with schema",
2006+
),
2007+
StringifiedPlan::new(PlanType::PhysicalPlanError, "physical plan error"),
2008+
];
2009+
let explain = Arc::new(ExplainExec::new(
2010+
Arc::clone(&schema),
2011+
stringified_plans.clone(),
2012+
true,
2013+
));
2014+
2015+
let ctx = SessionContext::new();
2016+
let roundtripped = roundtrip_test_and_return(
2017+
explain,
2018+
&ctx,
2019+
&DefaultPhysicalExtensionCodec {},
2020+
&DefaultPhysicalProtoConverter {},
2021+
)?;
2022+
let roundtripped = roundtripped.downcast_ref::<ExplainExec>().unwrap();
2023+
2024+
assert_eq!(roundtripped.schema(), schema);
2025+
assert_eq!(roundtripped.stringified_plans(), stringified_plans);
2026+
assert!(roundtripped.verbose());
2027+
Ok(())
2028+
}
2029+
19592030
#[tokio::test]
19602031
async fn roundtrip_json_source() -> Result<()> {
19612032
let ctx = SessionContext::new();

0 commit comments

Comments
 (0)