Skip to content

Commit f719174

Browse files
adriangbclaude
authored andcommitted
Proto: migrate MemorySourceConfig to per-source try_to_proto / try_from_proto hooks (apache#24187)
## Which issue does this PR close? - Closes apache#23518. - Part of apache#23494. ## Rationale for this change Continues the per-plan proto hook migration (apache#23494, pattern established in apache#23495 and apache#23683): each plan/source owns its wire logic via `try_to_proto` / `try_from_proto` hooks instead of the central downcast/match chains in `datafusion-proto`. ## What changes are included in this PR? - `DataSource::try_to_proto` implemented for `MemorySourceConfig` and inherent `MemorySourceConfig::try_from_proto` added in `datafusion/datasource/src/memory.rs` (behind the `proto` feature). The record-batch IPC serde is pure Arrow and is inlined locally. - The `MemoryScan` decode arm in `datafusion-proto` now delegates to `MemorySourceConfig::try_from_proto`, and the central `MemoryScan` encode arm in `try_from_data_source_exec` is **deleted** — proof that the hook is the only path. - `try_into_memory_scan_physical_plan` becomes a deprecated shim delegating to the new hook; the now-unused `serialize_record_batches` / `parse_record_batches` helpers are deprecated. The wire format is byte-for-byte identical: I verified locally that a memory scan plan (with sort information, fetch, and `show_sizes=false`) encodes to identical bytes before and after this change, and that bytes encoded by each version decode correctly with the other. ## Are these changes tested? Yes. Existing roundtrip tests (`roundtrip_memory_source`, `roundtrip_memory_source_empty_projection`) now exercise the hooks since the central arms are gone. A new test `roundtrip_memory_source_sort_information_and_fetch` covers `sort_information`, `fetch`, and `show_sizes`, asserting on the decoded `MemorySourceConfig` directly rather than only the display string. ## Are there any user-facing changes? No behavior changes. `PhysicalPlanNodeExt::try_into_memory_scan_physical_plan`, `parse_record_batches`, and `serialize_record_batches` are deprecated (still functional). 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Fable 5 <[email protected]>
1 parent d021cd9 commit f719174

5 files changed

Lines changed: 220 additions & 89 deletions

File tree

‎datafusion/datasource/src/memory.rs‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,58 @@ impl DataSource for MemorySourceConfig {
256256
})
257257
.transpose()
258258
}
259+
260+
/// Serialize this `MemorySourceConfig` as a `MemoryScanExecNode` wrapped
261+
/// in a [`PhysicalPlanNode`]. Byte-compatible with the former central
262+
/// `MemoryScan` arm in `datafusion-proto`.
263+
///
264+
/// [`PhysicalPlanNode`]: datafusion_proto_models::protobuf::PhysicalPlanNode
265+
#[cfg(feature = "proto")]
266+
fn try_to_proto(
267+
&self,
268+
ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
269+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
270+
use datafusion_physical_expr_common::sort_expr::sort_exprs_try_to_proto;
271+
use datafusion_proto_models::protobuf;
272+
273+
let partitions = self
274+
.partitions
275+
.iter()
276+
.map(|batches| record_batches_to_ipc_bytes(batches))
277+
.collect::<Result<Vec<_>>>()?;
278+
279+
// Proto3 can't tell `None` from `Some(vec![])`; encode the latter
280+
// as the `[u32::MAX]` sentinel, matching the join/filter nodes.
281+
let projection = match self.projection.as_ref() {
282+
None => Vec::new(),
283+
Some(v) if v.is_empty() => vec![u32::MAX],
284+
Some(v) => v.iter().map(|x| *x as u32).collect(),
285+
};
286+
287+
let mut sort_information = Vec::with_capacity(self.sort_information.len());
288+
for ordering in &self.sort_information {
289+
let physical_sort_expr_nodes =
290+
sort_exprs_try_to_proto(ordering.iter(), &ctx.expr_ctx())?;
291+
sort_information.push(protobuf::PhysicalSortExprNodeCollection {
292+
physical_sort_expr_nodes,
293+
});
294+
}
295+
296+
Ok(Some(protobuf::PhysicalPlanNode {
297+
physical_plan_type: Some(
298+
protobuf::physical_plan_node::PhysicalPlanType::MemoryScan(
299+
protobuf::MemoryScanExecNode {
300+
partitions,
301+
schema: Some(self.schema.as_ref().try_into()?),
302+
projection,
303+
sort_information,
304+
show_sizes: self.show_sizes,
305+
fetch: self.fetch.map(|f| f as u32),
306+
},
307+
),
308+
),
309+
}))
310+
}
259311
}
260312

261313
impl MemorySourceConfig {
@@ -607,6 +659,96 @@ impl MemorySourceConfig {
607659
}
608660
}
609661

662+
#[cfg(feature = "proto")]
663+
impl MemorySourceConfig {
664+
/// Reconstruct a [`DataSourceExec`] wrapping a `MemorySourceConfig` from
665+
/// its protobuf representation. Byte-compatible with the former central
666+
/// `MemoryScan` arm in `datafusion-proto`.
667+
pub fn try_from_proto(
668+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
669+
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
670+
) -> Result<Arc<dyn datafusion_physical_plan::ExecutionPlan>> {
671+
use datafusion_common::internal_datafusion_err;
672+
use datafusion_physical_expr_common::sort_expr::sort_exprs_try_from_proto;
673+
use datafusion_proto_models::protobuf;
674+
675+
let scan = datafusion_physical_plan::expect_plan_variant!(
676+
node,
677+
protobuf::physical_plan_node::PhysicalPlanType::MemoryScan,
678+
"MemorySourceConfig",
679+
);
680+
681+
let partitions = scan
682+
.partitions
683+
.iter()
684+
.map(|buf| record_batches_from_ipc_bytes(buf))
685+
.collect::<Result<Vec<_>>>()?;
686+
687+
let proto_schema = scan.schema.as_ref().ok_or_else(|| {
688+
internal_datafusion_err!("schema in MemoryScanExecNode is missing.")
689+
})?;
690+
let schema: SchemaRef = SchemaRef::new(proto_schema.try_into()?);
691+
692+
// Preserve the empty-projection sentinel written by `try_to_proto`.
693+
let projection = match scan.projection.as_slice() {
694+
[] => None,
695+
[u32::MAX] => Some(Vec::new()),
696+
indices => Some(indices.iter().map(|i| *i as usize).collect()),
697+
};
698+
699+
let mut sort_information = vec![];
700+
for ordering in &scan.sort_information {
701+
let sort_exprs = sort_exprs_try_from_proto(
702+
&ordering.physical_sort_expr_nodes,
703+
&ctx.expr_ctx(&schema),
704+
)?;
705+
sort_information.extend(LexOrdering::new(sort_exprs));
706+
}
707+
708+
let source = Self::try_new(&partitions, schema, projection)?
709+
.with_limit(scan.fetch.map(|f| f as usize))
710+
.with_show_sizes(scan.show_sizes)
711+
.try_with_sort_information(sort_information)?;
712+
713+
Ok(DataSourceExec::from_data_source(source))
714+
}
715+
}
716+
717+
/// Encode record batches as Arrow IPC stream bytes; an empty slice encodes to
718+
/// an empty buffer.
719+
#[cfg(feature = "proto")]
720+
fn record_batches_to_ipc_bytes(batches: &[RecordBatch]) -> Result<Vec<u8>> {
721+
use arrow::ipc::writer::StreamWriter;
722+
723+
if batches.is_empty() {
724+
return Ok(vec![]);
725+
}
726+
let schema = batches[0].schema();
727+
let mut buf = Vec::new();
728+
let mut writer = StreamWriter::try_new(&mut buf, &schema)?;
729+
for batch in batches {
730+
writer.write(batch)?;
731+
}
732+
writer.finish()?;
733+
Ok(buf)
734+
}
735+
736+
/// Inverse of [`record_batches_to_ipc_bytes`].
737+
#[cfg(feature = "proto")]
738+
fn record_batches_from_ipc_bytes(buf: &[u8]) -> Result<Vec<RecordBatch>> {
739+
use arrow::ipc::reader::StreamReader;
740+
741+
if buf.is_empty() {
742+
return Ok(vec![]);
743+
}
744+
let reader = StreamReader::try_new(buf, None)?;
745+
let mut batches = Vec::new();
746+
for batch in reader {
747+
batches.push(batch?);
748+
}
749+
Ok(batches)
750+
}
751+
610752
/// For use in repartitioning, track the total size and original partition index.
611753
///
612754
/// Do not implement clone, in order to avoid unnecessary copying during repartitioning.

‎datafusion/proto/src/physical_plan/from_proto.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -459,6 +459,10 @@ pub fn parse_protobuf_file_scan_config(
459459
)
460460
}
461461

462+
#[deprecated(
463+
since = "55.0.0",
464+
note = "unused by DataFusion; `MemorySourceConfig` deserializes its record batches itself via `MemorySourceConfig::try_from_proto`"
465+
)]
462466
pub fn parse_record_batches(buf: &[u8]) -> Result<Vec<RecordBatch>> {
463467
if buf.is_empty() {
464468
return Ok(vec![]);

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 17 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ use datafusion_common::{
2828
};
2929
use datafusion_datasource::file_scan_config::FileScanConfig;
3030
use datafusion_datasource::sink::DataSinkExec;
31-
use datafusion_datasource::source::{DataSource, DataSourceExec};
31+
use datafusion_datasource::source::DataSourceExec;
3232
use datafusion_datasource_arrow::source::ArrowSource;
3333
#[cfg(feature = "avro")]
3434
use datafusion_datasource_avro::source::AvroSource;
@@ -46,7 +46,6 @@ use datafusion_expr::{AggregateUDF, HigherOrderUDF, ScalarUDF, WindowUDF};
4646
use datafusion_functions_table::generate_series::{
4747
Empty, GenSeriesArgs, GenerateSeriesTable, GenericSeriesState, TimestampValue,
4848
};
49-
use datafusion_physical_expr::LexOrdering;
5049
use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;
5150
use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;
5251
use datafusion_physical_plan::aggregates::AggregateExec;
@@ -88,12 +87,11 @@ use prost::bytes::BufMut;
8887

8988
use crate::convert_required;
9089
use crate::physical_plan::from_proto::{
91-
parse_physical_expr_with_converter, parse_physical_sort_exprs,
92-
parse_protobuf_file_scan_config, parse_record_batches, parse_table_schema_from_proto,
90+
parse_physical_expr_with_converter, parse_protobuf_file_scan_config,
91+
parse_table_schema_from_proto,
9392
};
9493
use crate::physical_plan::to_proto::{
9594
serialize_file_scan_config, serialize_physical_expr_with_converter,
96-
serialize_physical_sort_exprs, serialize_record_batches,
9795
};
9896
use crate::protobuf::physical_plan_node::PhysicalPlanType;
9997
use crate::protobuf::{self, SortMergeJoinExecNode, proto_error};
@@ -1095,8 +1093,8 @@ pub trait PhysicalPlanNodeExt: Sized {
10951093
PhysicalPlanType::AvroScan(scan) => {
10961094
self.try_into_avro_scan_physical_plan(scan, ctx, proto_converter)
10971095
}
1098-
PhysicalPlanType::MemoryScan(scan) => {
1099-
self.try_into_memory_scan_physical_plan(scan, ctx, proto_converter)
1096+
PhysicalPlanType::MemoryScan(_) => {
1097+
MemorySourceConfig::try_from_proto(self.node(), &decode_ctx)
11001098
}
11011099
PhysicalPlanType::ArrowScan(scan) => {
11021100
self.try_into_arrow_scan_physical_plan(scan, ctx, proto_converter)
@@ -1454,48 +1452,25 @@ pub trait PhysicalPlanNodeExt: Sized {
14541452
panic!("Unable to process a Avro PhysicalPlan when `avro` feature is not enabled")
14551453
}
14561454

1455+
#[deprecated(
1456+
since = "55.0.0",
1457+
note = "unused by DataFusion; `MemorySourceConfig` deserializes itself via `MemorySourceConfig::try_from_proto`"
1458+
)]
14571459
fn try_into_memory_scan_physical_plan(
14581460
&self,
14591461
scan: &protobuf::MemoryScanExecNode,
14601462
ctx: &PhysicalPlanDecodeContext<'_>,
14611463
proto_converter: &dyn PhysicalProtoConverterExtension,
14621464
) -> Result<Arc<dyn ExecutionPlan>> {
1463-
let partitions = scan
1464-
.partitions
1465-
.iter()
1466-
.map(|p| parse_record_batches(p))
1467-
.collect::<Result<Vec<_>>>()?;
1468-
1469-
let proto_schema = scan.schema.as_ref().ok_or_else(|| {
1470-
internal_datafusion_err!("schema in MemoryScanExecNode is missing.")
1471-
})?;
1472-
let schema: SchemaRef = SchemaRef::new(proto_schema.try_into()?);
1473-
1474-
// Preserve the empty-projection sentinel written by `try_from_data_source_exec`.
1475-
let projection = match scan.projection.as_slice() {
1476-
[] => None,
1477-
[u32::MAX] => Some(Vec::new()),
1478-
indices => Some(indices.iter().map(|i| *i as usize).collect()),
1465+
let node = protobuf::PhysicalPlanNode {
1466+
physical_plan_type: Some(PhysicalPlanType::MemoryScan(scan.clone())),
14791467
};
1480-
1481-
let mut sort_information = vec![];
1482-
for ordering in &scan.sort_information {
1483-
let sort_exprs = parse_physical_sort_exprs(
1484-
&ordering.physical_sort_expr_nodes,
1485-
ctx,
1486-
&schema,
1487-
proto_converter,
1488-
)?;
1489-
sort_information.extend(LexOrdering::new(sort_exprs));
1490-
}
1491-
1492-
let source = MemorySourceConfig::try_new(&partitions, schema, projection)?
1493-
.with_limit(scan.fetch.map(|f| f as usize))
1494-
.with_show_sizes(scan.show_sizes);
1495-
1496-
let source = source.try_with_sort_information(sort_information)?;
1497-
1498-
Ok(DataSourceExec::from_data_source(source))
1468+
let decoder = ConverterPlanDecoder {
1469+
ctx,
1470+
proto_converter,
1471+
};
1472+
let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
1473+
MemorySourceConfig::try_from_proto(&node, &decode_ctx)
14991474
}
15001475

15011476
#[deprecated(
@@ -2515,53 +2490,6 @@ pub trait PhysicalPlanNodeExt: Sized {
25152490
}
25162491
}
25172492

2518-
if let Some(source_conf) = data_source.downcast_ref::<MemorySourceConfig>() {
2519-
let proto_partitions = source_conf
2520-
.partitions()
2521-
.iter()
2522-
.map(|p| serialize_record_batches(p))
2523-
.collect::<Result<Vec<_>>>()?;
2524-
2525-
let proto_schema: protobuf::Schema =
2526-
source_conf.original_schema().as_ref().try_into()?;
2527-
2528-
// Proto3 can't tell `None` from `Some(vec![])`; encode the latter
2529-
// as the `[u32::MAX]` sentinel, matching the join/filter nodes.
2530-
let proto_projection = match source_conf.projection().as_ref() {
2531-
None => Vec::new(),
2532-
Some(v) if v.is_empty() => vec![u32::MAX],
2533-
Some(v) => v.iter().map(|x| *x as u32).collect(),
2534-
};
2535-
2536-
let proto_sort_information = source_conf
2537-
.sort_information()
2538-
.iter()
2539-
.map(|ordering| {
2540-
let sort_exprs = serialize_physical_sort_exprs(
2541-
ordering.to_owned(),
2542-
codec,
2543-
proto_converter,
2544-
)?;
2545-
Ok::<_, DataFusionError>(protobuf::PhysicalSortExprNodeCollection {
2546-
physical_sort_expr_nodes: sort_exprs,
2547-
})
2548-
})
2549-
.collect::<Result<Vec<_>, _>>()?;
2550-
2551-
return Ok(Some(protobuf::PhysicalPlanNode {
2552-
physical_plan_type: Some(PhysicalPlanType::MemoryScan(
2553-
protobuf::MemoryScanExecNode {
2554-
partitions: proto_partitions,
2555-
schema: Some(proto_schema),
2556-
projection: proto_projection,
2557-
sort_information: proto_sort_information,
2558-
show_sizes: source_conf.show_sizes(),
2559-
fetch: source_conf.fetch().map(|f| f as u32),
2560-
},
2561-
)),
2562-
}));
2563-
}
2564-
25652493
Ok(None)
25662494
}
25672495

‎datafusion/proto/src/physical_plan/to_proto.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -427,6 +427,10 @@ pub fn serialize_maybe_filter(
427427
}
428428
}
429429

430+
#[deprecated(
431+
since = "55.0.0",
432+
note = "unused by DataFusion; `MemorySourceConfig` serializes its record batches itself via `DataSource::try_to_proto`"
433+
)]
430434
pub fn serialize_record_batches(batches: &[RecordBatch]) -> Result<Vec<u8>> {
431435
if batches.is_empty() {
432436
return Ok(vec![]);

‎datafusion/proto/tests/cases/roundtrip_physical_plan.rs‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3526,6 +3526,59 @@ async fn roundtrip_memory_source() -> Result<()> {
35263526
roundtrip_test(plan)
35273527
}
35283528

3529+
#[tokio::test]
3530+
async fn roundtrip_memory_source_sort_information_and_fetch() -> Result<()> {
3531+
use datafusion::datasource::memory::MemorySourceConfig;
3532+
use datafusion::datasource::source::DataSource as _;
3533+
3534+
let schema = Arc::new(Schema::new(vec![
3535+
Field::new("a", DataType::Utf8, false),
3536+
Field::new("b", DataType::Int64, false),
3537+
]));
3538+
let batch = RecordBatch::try_new(
3539+
Arc::clone(&schema),
3540+
vec![
3541+
Arc::new(arrow::array::StringArray::from(vec!["Tom", "Bob"])),
3542+
Arc::new(arrow::array::Int64Array::from(vec![18i64, 21i64])),
3543+
],
3544+
)?;
3545+
let ordering = LexOrdering::new(vec![PhysicalSortExpr::new(
3546+
col("b", &schema)?,
3547+
SortOptions {
3548+
descending: true,
3549+
nulls_first: false,
3550+
},
3551+
)])
3552+
.unwrap();
3553+
let source = MemorySourceConfig::try_new(&[vec![batch]], Arc::clone(&schema), None)?
3554+
.with_limit(Some(1))
3555+
.with_show_sizes(false)
3556+
.try_with_sort_information(vec![ordering])?;
3557+
let exec_plan = DataSourceExec::from_data_source(source.clone());
3558+
3559+
let ctx = SessionContext::new();
3560+
let codec = DefaultPhysicalExtensionCodec {};
3561+
let proto_converter = DefaultPhysicalProtoConverter {};
3562+
let decoded = roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;
3563+
3564+
// The string representation does not include every field; check the
3565+
// decoded source directly.
3566+
let decoded = decoded
3567+
.downcast_ref::<DataSourceExec>()
3568+
.expect("expected DataSourceExec");
3569+
let decoded_source = decoded
3570+
.data_source()
3571+
.downcast_ref::<MemorySourceConfig>()
3572+
.expect("expected MemorySourceConfig");
3573+
assert_eq!(decoded_source.partitions(), source.partitions());
3574+
assert_eq!(decoded_source.original_schema(), source.original_schema());
3575+
assert_eq!(decoded_source.projection(), source.projection());
3576+
assert_eq!(decoded_source.sort_information(), source.sort_information());
3577+
assert_eq!(decoded_source.fetch(), Some(1));
3578+
assert!(!decoded_source.show_sizes());
3579+
Ok(())
3580+
}
3581+
35293582
#[tokio::test]
35303583
async fn roundtrip_listing_table_with_schema_metadata() -> Result<()> {
35313584
let ctx = SessionContext::new();

0 commit comments

Comments
 (0)