Skip to content

Commit dc69242

Browse files
buraksennkosiew
authored andcommitted
refactor(proto): migrate ArrowSource serde (apache#24189)
## Which issue does this PR close? - Part of apache#23516. ## Rationale for this change Part of epic apache#23494. Moves `ArrowSource` protobuf serialization from the central dispatch into the source implementation. ## What changes are included in this PR? Add protobuf serialization and deserialization to `ArrowSource` and add the required `proto` feature wiring to `datafusion-datasource-arrow`. Repoint the live decode arm to `ArrowSource::try_from_proto` and remove the old central encode arm. Keep `try_into_arrow_scan_physical_plan` as a deprecated compatibility wrapper that delegates to the new implementation. The protobuf wire format remains unchanged. ## Are these changes tested? Yes. The existing `roundtrip_arrow_scan` coverage passes through the new hooks. ## Are there any user-facing changes? The existing `PhysicalPlanNodeExt` method remains available as a deprecated compatibility wrapper. There is no immediate API removal or wire-format change.
1 parent a66175e commit dc69242

5 files changed

Lines changed: 96 additions & 51 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/datasource-arrow/Cargo.toml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ datafusion-execution = { workspace = true }
4242
datafusion-expr = { workspace = true }
4343
datafusion-physical-expr-common = { workspace = true }
4444
datafusion-physical-plan = { workspace = true }
45+
datafusion-proto-models = { workspace = true, optional = true }
4546
datafusion-session = { workspace = true }
4647
futures = { workspace = true }
4748
itertools = { workspace = true }
@@ -65,3 +66,10 @@ path = "src/mod.rs"
6566
# This feature is deprecated, as core functionality in the SpillManager requires all features
6667
# it enabled, and will be removed in a future version.
6768
compression = []
69+
# Enables `FileSource::try_to_proto` on `ArrowSource` and the `ArrowScan` decode
70+
# entry point. Mirrors the `proto` feature on `datafusion-datasource`.
71+
proto = [
72+
"dep:datafusion-proto-models",
73+
"datafusion-datasource/proto",
74+
"datafusion-physical-plan/proto",
75+
]

‎datafusion/datasource-arrow/src/source.rs‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,64 @@ impl FileSource for ArrowSource {
392392
fn projection(&self) -> Option<&ProjectionExprs> {
393393
Some(&self.projection.source)
394394
}
395+
396+
/// Emit an `ArrowScan` node wrapping the shared base config.
397+
///
398+
/// Decoding defaults to the IPC file format because protobuf does not
399+
/// distinguish it from the IPC stream format.
400+
#[cfg(feature = "proto")]
401+
fn try_to_proto(
402+
&self,
403+
base: &FileScanConfig,
404+
ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
405+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
406+
use datafusion_proto_models::protobuf;
407+
use protobuf::physical_plan_node::PhysicalPlanType;
408+
409+
Ok(Some(protobuf::PhysicalPlanNode {
410+
physical_plan_type: Some(PhysicalPlanType::ArrowScan(
411+
protobuf::ArrowScanExecNode {
412+
base_conf: Some(base.try_to_proto(ctx)?),
413+
},
414+
)),
415+
}))
416+
}
417+
}
418+
419+
#[cfg(feature = "proto")]
420+
impl ArrowSource {
421+
/// Reconstructs a `DataSourceExec` from a protobuf `ArrowScan`.
422+
///
423+
/// Defaults to the IPC file format because protobuf does not distinguish it
424+
/// from the IPC stream format.
425+
pub fn try_from_proto(
426+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
427+
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
428+
) -> Result<Arc<dyn datafusion_physical_plan::ExecutionPlan>> {
429+
use datafusion_datasource::file_scan_config::FileScanConfig;
430+
use datafusion_datasource::source::DataSourceExec;
431+
use datafusion_proto_models::protobuf;
432+
433+
let scan = match &node.physical_plan_type {
434+
Some(protobuf::physical_plan_node::PhysicalPlanType::ArrowScan(scan)) => scan,
435+
_ => {
436+
return datafusion_common::internal_err!(
437+
"PhysicalPlanNode is not an ArrowScan"
438+
);
439+
}
440+
};
441+
442+
let base_conf = scan.base_conf.as_ref().ok_or_else(|| {
443+
datafusion_common::internal_datafusion_err!(
444+
"ArrowScanExecNode is missing required field 'base_conf'"
445+
)
446+
})?;
447+
448+
let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?;
449+
let source = Arc::new(ArrowSource::new_file_source(table_schema));
450+
let scan_conf = FileScanConfig::try_from_proto(base_conf, ctx, source)?;
451+
Ok(DataSourceExec::from_data_source(scan_conf))
452+
}
395453
}
396454

397455
/// `FileOpener` wrapper for both Arrow IPC file and stream formats

‎datafusion/proto/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ datafusion-catalog = { workspace = true }
6262
datafusion-catalog-listing = { workspace = true }
6363
datafusion-common = { workspace = true }
6464
datafusion-datasource = { workspace = true, features = ["proto"] }
65-
datafusion-datasource-arrow = { workspace = true }
65+
datafusion-datasource-arrow = { workspace = true, features = ["proto"] }
6666
datafusion-datasource-avro = { workspace = true, optional = true, features = ["proto"] }
6767
datafusion-datasource-csv = { workspace = true, features = ["proto"] }
6868
datafusion-datasource-json = { workspace = true, features = ["proto"] }

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 28 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ use datafusion_catalog::memory::MemorySourceConfig;
2626
use datafusion_common::{
2727
DataFusionError, Result, internal_datafusion_err, internal_err, not_impl_err,
2828
};
29-
use datafusion_datasource::file_scan_config::FileScanConfig;
3029
use datafusion_datasource::sink::DataSinkExec;
3130
use datafusion_datasource::source::DataSourceExec;
3231
use datafusion_datasource_arrow::source::ArrowSource;
@@ -86,13 +85,8 @@ use prost::Message;
8685
use prost::bytes::BufMut;
8786

8887
use crate::convert_required;
89-
use crate::physical_plan::from_proto::{
90-
parse_physical_expr_with_converter, parse_protobuf_file_scan_config,
91-
parse_table_schema_from_proto,
92-
};
93-
use crate::physical_plan::to_proto::{
94-
serialize_file_scan_config, serialize_physical_expr_with_converter,
95-
};
88+
use crate::physical_plan::from_proto::parse_physical_expr_with_converter;
89+
use crate::physical_plan::to_proto::serialize_physical_expr_with_converter;
9690
use crate::protobuf::physical_plan_node::PhysicalPlanType;
9791
use crate::protobuf::{self, SortMergeJoinExecNode, proto_error};
9892

@@ -115,7 +109,9 @@ mod file_scan_config_serde {
115109
use datafusion_common::{Constraint, Constraints, ScalarValue, Statistics};
116110
use datafusion_datasource::file::FileSource;
117111
use datafusion_datasource::file_groups::FileGroup;
118-
use datafusion_datasource::file_scan_config::FileScanConfigBuilder;
112+
use datafusion_datasource::file_scan_config::{
113+
FileScanConfig, FileScanConfigBuilder,
114+
};
119115
use datafusion_datasource::file_stream::FileOpener;
120116
use datafusion_datasource::{PartitionedFile, TableSchema};
121117
use datafusion_execution::object_store::ObjectStoreUrl;
@@ -1103,8 +1099,8 @@ pub trait PhysicalPlanNodeExt: Sized {
11031099
PhysicalPlanType::MemoryScan(_) => {
11041100
MemorySourceConfig::try_from_proto(self.node(), &decode_ctx)
11051101
}
1106-
PhysicalPlanType::ArrowScan(scan) => {
1107-
self.try_into_arrow_scan_physical_plan(scan, ctx, proto_converter)
1102+
PhysicalPlanType::ArrowScan(_) => {
1103+
ArrowSource::try_from_proto(self.node(), &decode_ctx)
11081104
}
11091105
#[expect(
11101106
deprecated,
@@ -1233,16 +1229,6 @@ pub trait PhysicalPlanNodeExt: Sized {
12331229
return Ok(node);
12341230
}
12351231

1236-
if let Some(data_source_exec) = plan.downcast_ref::<DataSourceExec>()
1237-
&& let Some(node) = protobuf::PhysicalPlanNode::try_from_data_source_exec(
1238-
data_source_exec,
1239-
codec,
1240-
proto_converter,
1241-
)?
1242-
{
1243-
return Ok(node);
1244-
}
1245-
12461232
if let Some(exec) = plan.downcast_ref::<LazyMemoryExec>()
12471233
&& let Some(node) =
12481234
protobuf::PhysicalPlanNode::try_from_lazy_memory_exec(exec)?
@@ -1386,23 +1372,25 @@ pub trait PhysicalPlanNodeExt: Sized {
13861372
JsonSource::try_from_proto(&node, &decode_ctx)
13871373
}
13881374

1375+
#[deprecated(
1376+
since = "55.0.0",
1377+
note = "unused by DataFusion; `ArrowSource` deserializes itself via `ArrowSource::try_from_proto`"
1378+
)]
13891379
fn try_into_arrow_scan_physical_plan(
13901380
&self,
13911381
scan: &protobuf::ArrowScanExecNode,
13921382
ctx: &PhysicalPlanDecodeContext<'_>,
13931383
proto_converter: &dyn PhysicalProtoConverterExtension,
13941384
) -> Result<Arc<dyn ExecutionPlan>> {
1395-
let base_conf = scan.base_conf.as_ref().ok_or_else(|| {
1396-
internal_datafusion_err!("base_conf in ArrowScanExecNode is missing.")
1397-
})?;
1398-
let table_schema = parse_table_schema_from_proto(base_conf)?;
1399-
let scan_conf = parse_protobuf_file_scan_config(
1400-
base_conf,
1385+
let node = protobuf::PhysicalPlanNode {
1386+
physical_plan_type: Some(PhysicalPlanType::ArrowScan(scan.clone())),
1387+
};
1388+
let decoder = ConverterPlanDecoder {
14011389
ctx,
14021390
proto_converter,
1403-
Arc::new(ArrowSource::new_file_source(table_schema)),
1404-
)?;
1405-
Ok(DataSourceExec::from_data_source(scan_conf))
1391+
};
1392+
let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
1393+
ArrowSource::try_from_proto(&node, &decode_ctx)
14061394
}
14071395

14081396
#[cfg_attr(not(feature = "parquet"), expect(unused_variables))]
@@ -2459,31 +2447,21 @@ pub trait PhysicalPlanNodeExt: Sized {
24592447
})
24602448
}
24612449

2450+
#[deprecated(
2451+
since = "55.0.0",
2452+
note = "unused by DataFusion; `DataSourceExec` serializes itself via `ExecutionPlan::try_to_proto`"
2453+
)]
24622454
fn try_from_data_source_exec(
24632455
data_source_exec: &DataSourceExec,
24642456
codec: &dyn PhysicalExtensionCodec,
24652457
proto_converter: &dyn PhysicalProtoConverterExtension,
24662458
) -> Result<Option<protobuf::PhysicalPlanNode>> {
2467-
let data_source = data_source_exec.data_source();
2468-
2469-
if let Some(scan_conf) = data_source.downcast_ref::<FileScanConfig>() {
2470-
let source = scan_conf.file_source();
2471-
if let Some(_arrow_source) = source.downcast_ref::<ArrowSource>() {
2472-
return Ok(Some(protobuf::PhysicalPlanNode {
2473-
physical_plan_type: Some(PhysicalPlanType::ArrowScan(
2474-
protobuf::ArrowScanExecNode {
2475-
base_conf: Some(serialize_file_scan_config(
2476-
scan_conf,
2477-
codec,
2478-
proto_converter,
2479-
)?),
2480-
},
2481-
)),
2482-
}));
2483-
}
2484-
}
2485-
2486-
Ok(None)
2459+
let encoder = ConverterPlanEncoder {
2460+
codec,
2461+
proto_converter,
2462+
};
2463+
let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder);
2464+
data_source_exec.try_to_proto(&encode_ctx)
24872465
}
24882466

24892467
#[deprecated(

0 commit comments

Comments
 (0)