Skip to content

Commit a27bc6c

Browse files
committed
refactor(proto): migrate ExplainExec serde
Move ExplainExec serialization into its execution plan implementation. This removes its live central dispatch while preserving the existing wire format and deprecated compatibility entry points. Add roundtrip coverage for every StringifiedPlan variant and all stored ExplainExec fields. Refs apache#23511 Signed-off-by: Jiawei Zhao <[email protected]>
1 parent 16471ee commit a27bc6c

3 files changed

Lines changed: 281 additions & 32 deletions

File tree

‎datafusion/physical-plan/src/explain.rs‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,188 @@ impl ExecutionPlan for ExplainExec {
185185
futures::stream::iter(vec![Ok(record_batch)]),
186186
)))
187187
}
188+
189+
#[cfg(feature = "proto")]
190+
fn try_to_proto(
191+
&self,
192+
_ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
193+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
194+
use datafusion_proto_models::protobuf;
195+
196+
Ok(Some(protobuf::PhysicalPlanNode {
197+
physical_plan_type: Some(
198+
protobuf::physical_plan_node::PhysicalPlanType::Explain(
199+
protobuf::ExplainExecNode {
200+
schema: Some(self.schema().as_ref().try_into()?),
201+
stringified_plans: self
202+
.stringified_plans()
203+
.iter()
204+
.map(stringified_plan_to_proto)
205+
.collect(),
206+
verbose: self.verbose(),
207+
},
208+
),
209+
),
210+
}))
211+
}
212+
}
213+
214+
#[cfg(feature = "proto")]
215+
impl ExplainExec {
216+
/// Reconstruct an [`ExplainExec`] from its protobuf representation.
217+
pub fn try_from_proto(
218+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
219+
_ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
220+
) -> Result<Arc<dyn ExecutionPlan>> {
221+
use datafusion_proto_models::protobuf;
222+
223+
let explain = crate::expect_plan_variant!(
224+
node,
225+
protobuf::physical_plan_node::PhysicalPlanType::Explain,
226+
"ExplainExec",
227+
);
228+
let schema = explain.schema.as_ref().ok_or_else(|| {
229+
datafusion_common::internal_datafusion_err!(
230+
"ExplainExec is missing required field 'schema'"
231+
)
232+
})?;
233+
Ok(Arc::new(ExplainExec::new(
234+
Arc::new(arrow::datatypes::Schema::try_from(schema)?),
235+
explain
236+
.stringified_plans
237+
.iter()
238+
.map(stringified_plan_from_proto)
239+
.collect(),
240+
explain.verbose,
241+
)))
242+
}
243+
}
244+
245+
#[cfg(feature = "proto")]
246+
fn stringified_plan_to_proto(
247+
stringified_plan: &StringifiedPlan,
248+
) -> datafusion_proto_models::protobuf::StringifiedPlan {
249+
use datafusion_common::display::PlanType;
250+
use datafusion_proto_models::datafusion_common::EmptyMessage;
251+
use datafusion_proto_models::protobuf;
252+
use protobuf::plan_type::PlanTypeEnum::{
253+
AnalyzedLogicalPlan, FinalAnalyzedLogicalPlan, FinalLogicalPlan,
254+
FinalPhysicalPlan, FinalPhysicalPlanWithSchema, FinalPhysicalPlanWithStats,
255+
InitialLogicalPlan, InitialPhysicalPlan, InitialPhysicalPlanWithSchema,
256+
InitialPhysicalPlanWithStats, OptimizedLogicalPlan, OptimizedPhysicalPlan,
257+
PhysicalPlanError,
258+
};
259+
260+
protobuf::StringifiedPlan {
261+
plan_type: match stringified_plan.clone().plan_type {
262+
PlanType::InitialLogicalPlan => Some(protobuf::PlanType {
263+
plan_type_enum: Some(InitialLogicalPlan(EmptyMessage {})),
264+
}),
265+
PlanType::AnalyzedLogicalPlan { analyzer_name } => Some(protobuf::PlanType {
266+
plan_type_enum: Some(AnalyzedLogicalPlan(
267+
protobuf::AnalyzedLogicalPlanType { analyzer_name },
268+
)),
269+
}),
270+
PlanType::FinalAnalyzedLogicalPlan => Some(protobuf::PlanType {
271+
plan_type_enum: Some(FinalAnalyzedLogicalPlan(EmptyMessage {})),
272+
}),
273+
PlanType::OptimizedLogicalPlan { optimizer_name } => {
274+
Some(protobuf::PlanType {
275+
plan_type_enum: Some(OptimizedLogicalPlan(
276+
protobuf::OptimizedLogicalPlanType { optimizer_name },
277+
)),
278+
})
279+
}
280+
PlanType::FinalLogicalPlan => Some(protobuf::PlanType {
281+
plan_type_enum: Some(FinalLogicalPlan(EmptyMessage {})),
282+
}),
283+
PlanType::InitialPhysicalPlan => Some(protobuf::PlanType {
284+
plan_type_enum: Some(InitialPhysicalPlan(EmptyMessage {})),
285+
}),
286+
PlanType::OptimizedPhysicalPlan { optimizer_name } => {
287+
Some(protobuf::PlanType {
288+
plan_type_enum: Some(OptimizedPhysicalPlan(
289+
protobuf::OptimizedPhysicalPlanType { optimizer_name },
290+
)),
291+
})
292+
}
293+
PlanType::FinalPhysicalPlan => Some(protobuf::PlanType {
294+
plan_type_enum: Some(FinalPhysicalPlan(EmptyMessage {})),
295+
}),
296+
PlanType::InitialPhysicalPlanWithStats => Some(protobuf::PlanType {
297+
plan_type_enum: Some(InitialPhysicalPlanWithStats(EmptyMessage {})),
298+
}),
299+
PlanType::InitialPhysicalPlanWithSchema => Some(protobuf::PlanType {
300+
plan_type_enum: Some(InitialPhysicalPlanWithSchema(EmptyMessage {})),
301+
}),
302+
PlanType::FinalPhysicalPlanWithStats => Some(protobuf::PlanType {
303+
plan_type_enum: Some(FinalPhysicalPlanWithStats(EmptyMessage {})),
304+
}),
305+
PlanType::FinalPhysicalPlanWithSchema => Some(protobuf::PlanType {
306+
plan_type_enum: Some(FinalPhysicalPlanWithSchema(EmptyMessage {})),
307+
}),
308+
PlanType::PhysicalPlanError => Some(protobuf::PlanType {
309+
plan_type_enum: Some(PhysicalPlanError(EmptyMessage {})),
310+
}),
311+
},
312+
plan: stringified_plan.plan.to_string(),
313+
}
314+
}
315+
316+
#[cfg(feature = "proto")]
317+
fn stringified_plan_from_proto(
318+
stringified_plan: &datafusion_proto_models::protobuf::StringifiedPlan,
319+
) -> StringifiedPlan {
320+
use datafusion_common::display::PlanType;
321+
use datafusion_proto_models::protobuf::plan_type::PlanTypeEnum::{
322+
AnalyzedLogicalPlan, FinalAnalyzedLogicalPlan, FinalLogicalPlan,
323+
FinalPhysicalPlan, FinalPhysicalPlanWithSchema, FinalPhysicalPlanWithStats,
324+
InitialLogicalPlan, InitialPhysicalPlan, InitialPhysicalPlanWithSchema,
325+
InitialPhysicalPlanWithStats, OptimizedLogicalPlan, OptimizedPhysicalPlan,
326+
PhysicalPlanError,
327+
};
328+
use datafusion_proto_models::protobuf::{
329+
AnalyzedLogicalPlanType, OptimizedLogicalPlanType, OptimizedPhysicalPlanType,
330+
};
331+
332+
StringifiedPlan {
333+
plan_type: match stringified_plan
334+
.plan_type
335+
.as_ref()
336+
.and_then(|plan_type| plan_type.plan_type_enum.as_ref())
337+
.unwrap_or_else(|| {
338+
panic!(
339+
"Cannot create protobuf::StringifiedPlan from {stringified_plan:?}"
340+
)
341+
}) {
342+
InitialLogicalPlan(_) => PlanType::InitialLogicalPlan,
343+
AnalyzedLogicalPlan(AnalyzedLogicalPlanType { analyzer_name }) => {
344+
PlanType::AnalyzedLogicalPlan {
345+
analyzer_name: analyzer_name.clone(),
346+
}
347+
}
348+
FinalAnalyzedLogicalPlan(_) => PlanType::FinalAnalyzedLogicalPlan,
349+
OptimizedLogicalPlan(OptimizedLogicalPlanType { optimizer_name }) => {
350+
PlanType::OptimizedLogicalPlan {
351+
optimizer_name: optimizer_name.clone(),
352+
}
353+
}
354+
FinalLogicalPlan(_) => PlanType::FinalLogicalPlan,
355+
InitialPhysicalPlan(_) => PlanType::InitialPhysicalPlan,
356+
InitialPhysicalPlanWithStats(_) => PlanType::InitialPhysicalPlanWithStats,
357+
InitialPhysicalPlanWithSchema(_) => PlanType::InitialPhysicalPlanWithSchema,
358+
OptimizedPhysicalPlan(OptimizedPhysicalPlanType { optimizer_name }) => {
359+
PlanType::OptimizedPhysicalPlan {
360+
optimizer_name: optimizer_name.clone(),
361+
}
362+
}
363+
FinalPhysicalPlan(_) => PlanType::FinalPhysicalPlan,
364+
FinalPhysicalPlanWithStats(_) => PlanType::FinalPhysicalPlanWithStats,
365+
FinalPhysicalPlanWithSchema(_) => PlanType::FinalPhysicalPlanWithSchema,
366+
PhysicalPlanError(_) => PlanType::PhysicalPlanError,
367+
},
368+
plan: Arc::new(stringified_plan.plan.clone()),
369+
}
188370
}
189371

190372
/// If this plan should be shown, given the previous plan that was

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 28 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ use std::sync::Arc;
2424
use arrow::datatypes::{IntervalMonthDayNanoType, Schema, SchemaRef};
2525
use datafusion_catalog::memory::MemorySourceConfig;
2626
use datafusion_common::config::CsvOptions;
27-
use datafusion_common::display::StringifiedPlan;
2827
use datafusion_common::format::ExplainFormat;
2928
use datafusion_common::{
3029
DataFusionError, Result, internal_datafusion_err, internal_err, not_impl_err,
@@ -744,8 +743,8 @@ pub trait PhysicalPlanNodeExt: Sized {
744743
};
745744
let decode_ctx = ExecutionPlanDecodeCtx::new(&plan_decoder);
746745
match plan {
747-
PhysicalPlanType::Explain(explain) => {
748-
self.try_into_explain_physical_plan(explain, ctx, proto_converter)
746+
PhysicalPlanType::Explain(_) => {
747+
ExplainExec::try_from_proto(self.node(), &decode_ctx)
749748
}
750749
PhysicalPlanType::Projection(_) => {
751750
ProjectionExec::try_from_proto(self.node(), &decode_ctx)
@@ -894,10 +893,6 @@ pub trait PhysicalPlanNodeExt: Sized {
894893
return Ok(node);
895894
}
896895

897-
if let Some(exec) = plan.downcast_ref::<ExplainExec>() {
898-
return protobuf::PhysicalPlanNode::try_from_explain_exec(exec, codec);
899-
}
900-
901896
if let Some(exec) = plan.downcast_ref::<AnalyzeExec>() {
902897
return protobuf::PhysicalPlanNode::try_from_analyze_exec(
903898
exec,
@@ -993,21 +988,22 @@ pub trait PhysicalPlanNodeExt: Sized {
993988
}
994989
}
995990

991+
#[deprecated(
992+
since = "55.0.0",
993+
note = "unused by DataFusion; `ExplainExec` deserializes itself via `ExplainExec::try_from_proto`"
994+
)]
996995
fn try_into_explain_physical_plan(
997996
&self,
998-
explain: &protobuf::ExplainExecNode,
999-
_ctx: &PhysicalPlanDecodeContext<'_>,
1000-
_proto_converter: &dyn PhysicalProtoConverterExtension,
997+
_explain: &protobuf::ExplainExecNode,
998+
ctx: &PhysicalPlanDecodeContext<'_>,
999+
proto_converter: &dyn PhysicalProtoConverterExtension,
10011000
) -> Result<Arc<dyn ExecutionPlan>> {
1002-
Ok(Arc::new(ExplainExec::new(
1003-
Arc::new(explain.schema.as_ref().unwrap().try_into()?),
1004-
explain
1005-
.stringified_plans
1006-
.iter()
1007-
.map(StringifiedPlan::from_proto)
1008-
.collect(),
1009-
explain.verbose,
1010-
)))
1001+
let plan_decoder = ConverterPlanDecoder {
1002+
ctx,
1003+
proto_converter,
1004+
};
1005+
let decode_ctx = ExecutionPlanDecodeCtx::new(&plan_decoder);
1006+
ExplainExec::try_from_proto(self.node(), &decode_ctx)
10111007
}
10121008

10131009
#[deprecated(
@@ -2313,22 +2309,22 @@ pub trait PhysicalPlanNodeExt: Sized {
23132309
)))
23142310
}
23152311

2312+
#[deprecated(
2313+
since = "55.0.0",
2314+
note = "unused by DataFusion; `ExplainExec` serializes itself via `ExecutionPlan::try_to_proto`"
2315+
)]
23162316
fn try_from_explain_exec(
23172317
exec: &ExplainExec,
2318-
_codec: &dyn PhysicalExtensionCodec,
2318+
codec: &dyn PhysicalExtensionCodec,
23192319
) -> Result<protobuf::PhysicalPlanNode> {
2320-
Ok(protobuf::PhysicalPlanNode {
2321-
physical_plan_type: Some(PhysicalPlanType::Explain(
2322-
protobuf::ExplainExecNode {
2323-
schema: Some(exec.schema().as_ref().try_into()?),
2324-
stringified_plans: exec
2325-
.stringified_plans()
2326-
.iter()
2327-
.map(protobuf::StringifiedPlan::from_proto)
2328-
.collect(),
2329-
verbose: exec.verbose(),
2330-
},
2331-
)),
2320+
let proto_converter = DefaultPhysicalProtoConverter {};
2321+
let plan_encoder = ConverterPlanEncoder {
2322+
codec,
2323+
proto_converter: &proto_converter,
2324+
};
2325+
let encode_ctx = ExecutionPlanEncodeCtx::new(&plan_encoder);
2326+
exec.try_to_proto(&encode_ctx)?.ok_or_else(|| {
2327+
internal_datafusion_err!("ExplainExec did not serialize itself")
23322328
})
23332329
}
23342330

‎datafusion/proto/tests/cases/roundtrip_physical_plan.rs‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;
6666
use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
6767
use datafusion::physical_plan::coop::CooperativeExec;
6868
use datafusion::physical_plan::empty::EmptyExec;
69+
use datafusion::physical_plan::explain::ExplainExec;
6970
use datafusion::physical_plan::expressions::{
7071
BinaryExpr, Column, DynamicFilterPhysicalExpr, NotExpr, PhysicalSortExpr, binary,
7172
cast, col, in_list, like, lit,
@@ -98,6 +99,7 @@ use datafusion::physical_plan::{
9899
use datafusion::prelude::{ParquetReadOptions, SessionContext};
99100
use datafusion::scalar::ScalarValue;
100101
use datafusion_common::config::{ConfigOptions, TableParquetOptions};
102+
use datafusion_common::display::{PlanType, StringifiedPlan};
101103
use datafusion_common::file_options::csv_writer::CsvWriterOptions;
102104
use datafusion_common::file_options::json_writer::JsonWriterOptions;
103105
use datafusion_common::parsers::CompressionTypeVariant;
@@ -1957,6 +1959,75 @@ fn roundtrip_analyze() -> Result<()> {
19571959
))
19581960
}
19591961

1962+
#[test]
1963+
fn roundtrip_explain() -> Result<()> {
1964+
let schema = Arc::new(Schema::new(vec![
1965+
Field::new("plan_type", DataType::Utf8, false),
1966+
Field::new("plan", DataType::Utf8, false),
1967+
]));
1968+
let stringified_plans = vec![
1969+
StringifiedPlan::new(PlanType::InitialLogicalPlan, "initial logical"),
1970+
StringifiedPlan::new(
1971+
PlanType::AnalyzedLogicalPlan {
1972+
analyzer_name: "analyzer".to_string(),
1973+
},
1974+
"analyzed logical",
1975+
),
1976+
StringifiedPlan::new(PlanType::FinalAnalyzedLogicalPlan, "final analyzed"),
1977+
StringifiedPlan::new(
1978+
PlanType::OptimizedLogicalPlan {
1979+
optimizer_name: "logical optimizer".to_string(),
1980+
},
1981+
"optimized logical",
1982+
),
1983+
StringifiedPlan::new(PlanType::FinalLogicalPlan, "final logical"),
1984+
StringifiedPlan::new(PlanType::InitialPhysicalPlan, "initial physical"),
1985+
StringifiedPlan::new(
1986+
PlanType::InitialPhysicalPlanWithStats,
1987+
"initial physical with stats",
1988+
),
1989+
StringifiedPlan::new(
1990+
PlanType::InitialPhysicalPlanWithSchema,
1991+
"initial physical with schema",
1992+
),
1993+
StringifiedPlan::new(
1994+
PlanType::OptimizedPhysicalPlan {
1995+
optimizer_name: "physical optimizer".to_string(),
1996+
},
1997+
"optimized physical",
1998+
),
1999+
StringifiedPlan::new(PlanType::FinalPhysicalPlan, "final physical"),
2000+
StringifiedPlan::new(
2001+
PlanType::FinalPhysicalPlanWithStats,
2002+
"final physical with stats",
2003+
),
2004+
StringifiedPlan::new(
2005+
PlanType::FinalPhysicalPlanWithSchema,
2006+
"final physical with schema",
2007+
),
2008+
StringifiedPlan::new(PlanType::PhysicalPlanError, "physical plan error"),
2009+
];
2010+
let explain = Arc::new(ExplainExec::new(
2011+
Arc::clone(&schema),
2012+
stringified_plans.clone(),
2013+
true,
2014+
));
2015+
2016+
let ctx = SessionContext::new();
2017+
let roundtripped = roundtrip_test_and_return(
2018+
explain,
2019+
&ctx,
2020+
&DefaultPhysicalExtensionCodec {},
2021+
&DefaultPhysicalProtoConverter {},
2022+
)?;
2023+
let roundtripped = roundtripped.downcast_ref::<ExplainExec>().unwrap();
2024+
2025+
assert_eq!(roundtripped.schema(), schema);
2026+
assert_eq!(roundtripped.stringified_plans(), stringified_plans);
2027+
assert!(roundtripped.verbose());
2028+
Ok(())
2029+
}
2030+
19602031
#[tokio::test]
19612032
async fn roundtrip_json_source() -> Result<()> {
19622033
let ctx = SessionContext::new();

0 commit comments

Comments
 (0)