Skip to content

Commit a02a5a7

Browse files
committed
refactor(proto): migrate AnalyzeExec serde
Move AnalyzeExec serialization into its execution plan implementation. This removes its live central dispatch while preserving the existing wire format and deprecated compatibility entry points. Strengthen roundtrip coverage for the child plan, schema, flags, metric categories, and explain format. Closes apache#23511 Signed-off-by: Jiawei Zhao <[email protected]>
1 parent a27bc6c commit a02a5a7

3 files changed

Lines changed: 156 additions & 81 deletions

File tree

‎datafusion/physical-plan/src/analyze.rs‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,102 @@ impl ExecutionPlan for AnalyzeExec {
303303
futures::stream::once(output),
304304
)))
305305
}
306+
307+
#[cfg(feature = "proto")]
308+
fn try_to_proto(
309+
&self,
310+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
311+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
312+
use datafusion_proto_models::protobuf;
313+
314+
let input = ctx.encode_child(self.input())?;
315+
let (has_metric_categories, metric_categories) = match self.metric_categories() {
316+
Some(categories) => {
317+
(true, categories.iter().map(ToString::to_string).collect())
318+
}
319+
None => (false, vec![]),
320+
};
321+
let format = match self.format() {
322+
ExplainFormat::Indent => protobuf::ExplainFormat::Indent,
323+
ExplainFormat::Tree => protobuf::ExplainFormat::Tree,
324+
ExplainFormat::PostgresJSON => protobuf::ExplainFormat::Pgjson,
325+
ExplainFormat::Graphviz => protobuf::ExplainFormat::Graphviz,
326+
} as i32;
327+
Ok(Some(protobuf::PhysicalPlanNode {
328+
physical_plan_type: Some(
329+
protobuf::physical_plan_node::PhysicalPlanType::Analyze(Box::new(
330+
protobuf::AnalyzeExecNode {
331+
verbose: self.verbose(),
332+
show_statistics: self.show_statistics(),
333+
input: Some(Box::new(input)),
334+
schema: Some(self.schema().as_ref().try_into()?),
335+
has_metric_categories,
336+
metric_categories,
337+
format,
338+
},
339+
)),
340+
),
341+
}))
342+
}
343+
}
344+
345+
#[cfg(feature = "proto")]
346+
impl AnalyzeExec {
347+
/// Reconstruct an [`AnalyzeExec`] from its protobuf representation.
348+
pub fn try_from_proto(
349+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
350+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
351+
) -> Result<Arc<dyn ExecutionPlan>> {
352+
use datafusion_proto_models::protobuf;
353+
354+
let analyze = crate::expect_plan_variant!(
355+
node,
356+
protobuf::physical_plan_node::PhysicalPlanType::Analyze,
357+
"AnalyzeExec",
358+
);
359+
let input =
360+
ctx.decode_required_child(analyze.input.as_deref(), "AnalyzeExec", "input")?;
361+
let metric_categories = if analyze.has_metric_categories {
362+
Some(
363+
analyze
364+
.metric_categories
365+
.iter()
366+
.map(|category| category.parse::<MetricCategory>())
367+
.collect::<Result<Vec<_>>>()?,
368+
)
369+
} else {
370+
None
371+
};
372+
let proto_format =
373+
protobuf::ExplainFormat::try_from(analyze.format).map_err(|_| {
374+
DataFusionError::Internal(format!(
375+
"Received an AnalyzeExecNode message with unknown ExplainFormat {}",
376+
analyze.format
377+
))
378+
})?;
379+
let format = match proto_format {
380+
protobuf::ExplainFormat::Indent => ExplainFormat::Indent,
381+
protobuf::ExplainFormat::Tree => ExplainFormat::Tree,
382+
protobuf::ExplainFormat::Pgjson => ExplainFormat::PostgresJSON,
383+
protobuf::ExplainFormat::Graphviz => ExplainFormat::Graphviz,
384+
};
385+
let schema = analyze.schema.as_ref().ok_or_else(|| {
386+
datafusion_common::internal_datafusion_err!(
387+
"AnalyzeExec is missing required field 'schema'"
388+
)
389+
})?;
390+
Ok(Arc::new(
391+
AnalyzeExec::builder(
392+
analyze.verbose,
393+
analyze.show_statistics,
394+
input,
395+
Arc::new(arrow::datatypes::Schema::try_from(schema)?),
396+
)
397+
.with_metric_categories(metric_categories)
398+
.with_format(format)
399+
.build(),
400+
))
401+
}
306402
}
307403

308404
/// Creates the output of AnalyzeExec as a RecordBatch

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 22 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ use std::sync::Arc;
2424
use arrow::datatypes::{IntervalMonthDayNanoType, Schema, SchemaRef};
2525
use datafusion_catalog::memory::MemorySourceConfig;
2626
use datafusion_common::config::CsvOptions;
27-
use datafusion_common::format::ExplainFormat;
2827
use datafusion_common::{
2928
DataFusionError, Result, internal_datafusion_err, internal_err, not_impl_err,
3029
};
@@ -83,7 +82,6 @@ use datafusion_physical_plan::joins::{
8382
};
8483
use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
8584
use datafusion_physical_plan::memory::LazyMemoryExec;
86-
use datafusion_physical_plan::metrics::MetricCategory;
8785
use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;
8886
use datafusion_physical_plan::projection::ProjectionExec;
8987
use datafusion_physical_plan::proto::{
@@ -102,7 +100,7 @@ use prost::Message;
102100
use prost::bytes::BufMut;
103101

104102
use crate::common::{byte_to_string, str_to_byte};
105-
use crate::convert::{FromProto, TryFromProto};
103+
use crate::convert::TryFromProto;
106104
use crate::convert_required;
107105
use crate::physical_plan::from_proto::{
108106
parse_physical_expr_with_converter, parse_physical_sort_expr,
@@ -828,8 +826,8 @@ pub trait PhysicalPlanNodeExt: Sized {
828826
PhysicalPlanType::NestedLoopJoin(_) => {
829827
NestedLoopJoinExec::try_from_proto(self.node(), &decode_ctx)
830828
}
831-
PhysicalPlanType::Analyze(analyze) => {
832-
self.try_into_analyze_physical_plan(analyze, ctx, proto_converter)
829+
PhysicalPlanType::Analyze(_) => {
830+
AnalyzeExec::try_from_proto(self.node(), &decode_ctx)
833831
}
834832
PhysicalPlanType::JsonSink(sink) => {
835833
self.try_into_json_sink_physical_plan(sink, ctx, proto_converter)
@@ -893,14 +891,6 @@ pub trait PhysicalPlanNodeExt: Sized {
893891
return Ok(node);
894892
}
895893

896-
if let Some(exec) = plan.downcast_ref::<AnalyzeExec>() {
897-
return protobuf::PhysicalPlanNode::try_from_analyze_exec(
898-
exec,
899-
codec,
900-
proto_converter,
901-
);
902-
}
903-
904894
if let Some(exec) = plan.downcast_ref::<AggregateExec>() {
905895
return protobuf::PhysicalPlanNode::try_from_aggregate_exec(
906896
exec,
@@ -1924,48 +1914,22 @@ pub trait PhysicalPlanNodeExt: Sized {
19241914
NestedLoopJoinExec::try_from_proto(&node, &decode_ctx)
19251915
}
19261916

1917+
#[deprecated(
1918+
since = "55.0.0",
1919+
note = "unused by DataFusion; `AnalyzeExec` deserializes itself via `AnalyzeExec::try_from_proto`"
1920+
)]
19271921
fn try_into_analyze_physical_plan(
19281922
&self,
1929-
analyze: &protobuf::AnalyzeExecNode,
1923+
_analyze: &protobuf::AnalyzeExecNode,
19301924
ctx: &PhysicalPlanDecodeContext<'_>,
19311925
proto_converter: &dyn PhysicalProtoConverterExtension,
19321926
) -> Result<Arc<dyn ExecutionPlan>> {
1933-
let input: Arc<dyn ExecutionPlan> =
1934-
into_physical_plan(&analyze.input, ctx, proto_converter)?;
1935-
let metric_categories = if analyze.has_metric_categories {
1936-
let cats: Result<Vec<MetricCategory>> = analyze
1937-
.metric_categories
1938-
.iter()
1939-
.map(|s| s.parse::<MetricCategory>())
1940-
.collect();
1941-
Some(cats?)
1942-
} else {
1943-
None
1927+
let plan_decoder = ConverterPlanDecoder {
1928+
ctx,
1929+
proto_converter,
19441930
};
1945-
let pb_format =
1946-
protobuf::ExplainFormat::try_from(analyze.format).map_err(|_| {
1947-
DataFusionError::Internal(format!(
1948-
"Received an AnalyzeExecNode message with unknown ExplainFormat {}",
1949-
analyze.format
1950-
))
1951-
})?;
1952-
let format = match pb_format {
1953-
protobuf::ExplainFormat::Indent => ExplainFormat::Indent,
1954-
protobuf::ExplainFormat::Tree => ExplainFormat::Tree,
1955-
protobuf::ExplainFormat::Pgjson => ExplainFormat::PostgresJSON,
1956-
protobuf::ExplainFormat::Graphviz => ExplainFormat::Graphviz,
1957-
};
1958-
Ok(Arc::new(
1959-
AnalyzeExec::builder(
1960-
analyze.verbose,
1961-
analyze.show_statistics,
1962-
input,
1963-
Arc::new(convert_required!(analyze.schema)?),
1964-
)
1965-
.with_metric_categories(metric_categories)
1966-
.with_format(format)
1967-
.build(),
1968-
))
1931+
let decode_ctx = ExecutionPlanDecodeCtx::new(&plan_decoder);
1932+
AnalyzeExec::try_from_proto(self.node(), &decode_ctx)
19691933
}
19701934

19711935
fn try_into_json_sink_physical_plan(
@@ -2347,38 +2311,22 @@ pub trait PhysicalPlanNodeExt: Sized {
23472311
})
23482312
}
23492313

2314+
#[deprecated(
2315+
since = "55.0.0",
2316+
note = "unused by DataFusion; `AnalyzeExec` serializes itself via `ExecutionPlan::try_to_proto`"
2317+
)]
23502318
fn try_from_analyze_exec(
23512319
exec: &AnalyzeExec,
23522320
codec: &dyn PhysicalExtensionCodec,
23532321
proto_converter: &dyn PhysicalProtoConverterExtension,
23542322
) -> Result<protobuf::PhysicalPlanNode> {
2355-
let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
2356-
exec.input().to_owned(),
2323+
let plan_encoder = ConverterPlanEncoder {
23572324
codec,
23582325
proto_converter,
2359-
)?;
2360-
let (has_metric_categories, metric_categories) = match exec.metric_categories() {
2361-
Some(cats) => (true, cats.iter().map(|c| c.to_string()).collect()),
2362-
None => (false, vec![]),
2363-
};
2364-
let format = match exec.format() {
2365-
ExplainFormat::Indent => protobuf::ExplainFormat::Indent,
2366-
ExplainFormat::Tree => protobuf::ExplainFormat::Tree,
2367-
ExplainFormat::PostgresJSON => protobuf::ExplainFormat::Pgjson,
2368-
ExplainFormat::Graphviz => protobuf::ExplainFormat::Graphviz,
2369-
} as i32;
2370-
Ok(protobuf::PhysicalPlanNode {
2371-
physical_plan_type: Some(PhysicalPlanType::Analyze(Box::new(
2372-
protobuf::AnalyzeExecNode {
2373-
verbose: exec.verbose(),
2374-
show_statistics: exec.show_statistics(),
2375-
input: Some(Box::new(input)),
2376-
schema: Some(exec.schema().as_ref().try_into()?),
2377-
has_metric_categories,
2378-
metric_categories,
2379-
format,
2380-
},
2381-
))),
2326+
};
2327+
let encode_ctx = ExecutionPlanEncodeCtx::new(&plan_encoder);
2328+
exec.try_to_proto(&encode_ctx)?.ok_or_else(|| {
2329+
internal_datafusion_err!("AnalyzeExec did not serialize itself")
23822330
})
23832331
}
23842332

‎datafusion/proto/tests/cases/roundtrip_physical_plan.rs‎

Lines changed: 38 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ use datafusion::physical_plan::joins::{
7878
StreamJoinPartitionMode, SymmetricHashJoinExec,
7979
};
8080
use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
81+
use datafusion::physical_plan::metrics::MetricCategory;
8182
use datafusion::physical_plan::placeholder_row::PlaceholderRowExec;
8283
use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};
8384
use datafusion::physical_plan::repartition::RepartitionExec;
@@ -102,6 +103,7 @@ use datafusion_common::config::{ConfigOptions, TableParquetOptions};
102103
use datafusion_common::display::{PlanType, StringifiedPlan};
103104
use datafusion_common::file_options::csv_writer::CsvWriterOptions;
104105
use datafusion_common::file_options::json_writer::JsonWriterOptions;
106+
use datafusion_common::format::ExplainFormat;
105107
use datafusion_common::parsers::CompressionTypeVariant;
106108
use datafusion_common::stats::Precision;
107109
use datafusion_common::{
@@ -1949,14 +1951,43 @@ fn roundtrip_like() -> Result<()> {
19491951

19501952
#[test]
19511953
fn roundtrip_analyze() -> Result<()> {
1952-
let field_a = Field::new("plan_type", DataType::Utf8, false);
1953-
let field_b = Field::new("plan", DataType::Utf8, false);
1954-
let schema = Schema::new(vec![field_a, field_b]);
1955-
let input = Arc::new(PlaceholderRowExec::new(Arc::new(schema.clone())));
1954+
let schema = Arc::new(Schema::new(vec![
1955+
Field::new("plan_type", DataType::Utf8, false),
1956+
Field::new("plan", DataType::Utf8, false),
1957+
]));
1958+
let input = Arc::new(PlaceholderRowExec::new(Arc::clone(&schema)));
1959+
let metric_categories = vec![MetricCategory::Rows, MetricCategory::Timing];
1960+
let analyze = Arc::new(
1961+
AnalyzeExec::builder(true, true, input, Arc::clone(&schema))
1962+
.with_metric_categories(Some(metric_categories.clone()))
1963+
.with_format(ExplainFormat::Tree)
1964+
.build(),
1965+
);
19561966

1957-
roundtrip_test(Arc::new(
1958-
AnalyzeExec::builder(false, false, input, Arc::new(schema)).build(),
1959-
))
1967+
let ctx = SessionContext::new();
1968+
let roundtripped = roundtrip_test_and_return(
1969+
analyze,
1970+
&ctx,
1971+
&DefaultPhysicalExtensionCodec {},
1972+
&DefaultPhysicalProtoConverter {},
1973+
)?;
1974+
let roundtripped = roundtripped.downcast_ref::<AnalyzeExec>().unwrap();
1975+
1976+
assert_eq!(roundtripped.schema(), schema);
1977+
assert!(roundtripped.verbose());
1978+
assert!(roundtripped.show_statistics());
1979+
assert_eq!(
1980+
roundtripped.metric_categories(),
1981+
Some(metric_categories.as_slice())
1982+
);
1983+
assert_eq!(roundtripped.format(), &ExplainFormat::Tree);
1984+
assert!(
1985+
roundtripped
1986+
.input()
1987+
.downcast_ref::<PlaceholderRowExec>()
1988+
.is_some()
1989+
);
1990+
Ok(())
19601991
}
19611992

19621993
#[test]

0 commit comments

Comments
 (0)