Skip to content

Commit 853b07c

Browse files
adriangbclaude
andcommitted
prototype: migrate MemorySourceConfig; DataSourceExec fully on the hook (apache#22419)
MemorySourceConfig rides DataSource::try_to_proto (in datafusion-datasource, where the type actually lives). Record-batch IPC serde is pure Arrow, inlined locally like CsvSource's byte helpers — no codec. With every DataSource migrated (5 file formats + memory), try_from_data_source_exec and the DataSourceExec encode downcast arm are deleted entirely: DataSourceExec now serializes purely via the ExecutionPlan hook, delegating through DataSource -> FileSource with zero downcasts in the central encode path. roundtrip_memory_source + the 5 scan tests stay green (192 total). GenerateSeries/LazyMemoryExec remain typed arms (their generator types live in datafusion-functions-table, above physical-plan — the hook can't name them without a crate cycle). Extension stays as the terminal codec.try_encode fallback. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01SfBPC2yYCwHUn4V8pQCVj3
1 parent c6a26d8 commit 853b07c

2 files changed

Lines changed: 174 additions & 76 deletions

File tree

‎datafusion/datasource/src/memory.rs‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,94 @@ impl DataSource for MemorySourceConfig {
256256
})
257257
.transpose()
258258
}
259+
260+
/// Emit a `MemoryScan` node describing these in-memory partitions. Kept
261+
/// byte-identical to the former `try_from_data_source_exec`
262+
/// `MemorySourceConfig` branch in `datafusion-proto` (#22419).
263+
#[cfg(feature = "proto")]
264+
fn try_to_proto(
265+
&self,
266+
ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
267+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
268+
use datafusion_common::DataFusionError;
269+
use datafusion_proto_models::protobuf;
270+
use protobuf::physical_plan_node::PhysicalPlanType;
271+
272+
let partitions = self
273+
.partitions
274+
.iter()
275+
.map(|batches| proto_serialize_record_batches(batches))
276+
.collect::<Result<Vec<_>>>()?;
277+
278+
let projection = self.projection.as_ref().map_or_else(Vec::new, |v| {
279+
v.iter().map(|x| *x as u32).collect::<Vec<u32>>()
280+
});
281+
282+
let sort_information = self
283+
.sort_information
284+
.iter()
285+
.map(|ordering| {
286+
let physical_sort_expr_nodes = ordering
287+
.iter()
288+
.map(|sort_expr| {
289+
Ok::<_, DataFusionError>(protobuf::PhysicalSortExprNode {
290+
expr: Some(Box::new(ctx.encode_expr(&sort_expr.expr)?)),
291+
asc: !sort_expr.options.descending,
292+
nulls_first: sort_expr.options.nulls_first,
293+
})
294+
})
295+
.collect::<Result<Vec<_>>>()?;
296+
Ok::<_, DataFusionError>(protobuf::PhysicalSortExprNodeCollection {
297+
physical_sort_expr_nodes,
298+
})
299+
})
300+
.collect::<Result<Vec<_>>>()?;
301+
302+
Ok(Some(protobuf::PhysicalPlanNode {
303+
physical_plan_type: Some(PhysicalPlanType::MemoryScan(
304+
protobuf::MemoryScanExecNode {
305+
partitions,
306+
schema: Some((&*self.schema).try_into()?),
307+
projection,
308+
sort_information,
309+
show_sizes: self.show_sizes,
310+
fetch: self.fetch.map(|f| f as u32),
311+
},
312+
)),
313+
}))
314+
}
315+
}
316+
317+
/// Serialize a partition's record batches to Arrow IPC stream bytes. Local copy
318+
/// of `datafusion-proto`'s `serialize_record_batches`; kept byte-identical so the
319+
/// `MemoryScan` wire format is unchanged. The batch IPC bytes are fully portable
320+
/// (pure Arrow), so no `datafusion-proto` type is required here.
321+
#[cfg(feature = "proto")]
322+
fn proto_serialize_record_batches(batches: &[RecordBatch]) -> Result<Vec<u8>> {
323+
if batches.is_empty() {
324+
return Ok(vec![]);
325+
}
326+
let schema = batches[0].schema();
327+
let mut buf = Vec::new();
328+
let mut writer = arrow::ipc::writer::StreamWriter::try_new(&mut buf, &schema)?;
329+
for batch in batches {
330+
writer.write(batch)?;
331+
}
332+
writer.finish()?;
333+
Ok(buf)
334+
}
335+
336+
/// Parse a partition's Arrow IPC stream bytes back into record batches. Local
337+
/// copy of `datafusion-proto`'s `parse_record_batches`; kept byte-identical.
338+
#[cfg(feature = "proto")]
339+
fn proto_parse_record_batches(buf: &[u8]) -> Result<Vec<RecordBatch>> {
340+
if buf.is_empty() {
341+
return Ok(vec![]);
342+
}
343+
let reader = arrow::ipc::reader::StreamReader::try_new(buf, None)?;
344+
reader
345+
.map(|batch| batch.map_err(Into::into))
346+
.collect::<Result<Vec<_>>>()
259347
}
260348

261349
impl MemorySourceConfig {
@@ -607,6 +695,88 @@ impl MemorySourceConfig {
607695
}
608696
}
609697

698+
#[cfg(feature = "proto")]
699+
impl MemorySourceConfig {
700+
/// Reconstruct a `DataSourceExec` wrapping a `MemorySourceConfig` from a
701+
/// `MemoryScan` [`PhysicalPlanNode`]. The inverse of
702+
/// [`DataSource::try_to_proto`] on `MemorySourceConfig`; kept
703+
/// byte-compatible with the former `try_into_memory_scan_physical_plan` in
704+
/// `datafusion-proto` (#22419).
705+
///
706+
/// [`PhysicalPlanNode`]: datafusion_proto_models::protobuf::PhysicalPlanNode
707+
pub fn try_from_proto(
708+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
709+
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
710+
) -> Result<Arc<dyn datafusion_physical_plan::ExecutionPlan>> {
711+
use datafusion_common::{DataFusionError, internal_datafusion_err};
712+
use datafusion_physical_expr::PhysicalSortExpr;
713+
use datafusion_proto_models::protobuf;
714+
715+
let scan = match &node.physical_plan_type {
716+
Some(protobuf::physical_plan_node::PhysicalPlanType::MemoryScan(scan)) => {
717+
scan
718+
}
719+
_ => {
720+
return datafusion_common::internal_err!(
721+
"PhysicalPlanNode is not a MemoryScan"
722+
);
723+
}
724+
};
725+
726+
let partitions = scan
727+
.partitions
728+
.iter()
729+
.map(|p| proto_parse_record_batches(p))
730+
.collect::<Result<Vec<_>>>()?;
731+
732+
let proto_schema = scan.schema.as_ref().ok_or_else(|| {
733+
internal_datafusion_err!("schema in MemoryScanExecNode is missing.")
734+
})?;
735+
let schema: SchemaRef = SchemaRef::new(proto_schema.try_into()?);
736+
737+
let projection = if !scan.projection.is_empty() {
738+
Some(
739+
scan.projection
740+
.iter()
741+
.map(|i| *i as usize)
742+
.collect::<Vec<_>>(),
743+
)
744+
} else {
745+
None
746+
};
747+
748+
let mut sort_information = vec![];
749+
for ordering in &scan.sort_information {
750+
let sort_exprs = ordering
751+
.physical_sort_expr_nodes
752+
.iter()
753+
.map(|sort_expr| {
754+
let expr_node = sort_expr.expr.as_deref().ok_or_else(|| {
755+
internal_datafusion_err!(
756+
"PhysicalSortExprNode is missing required field 'expr'"
757+
)
758+
})?;
759+
Ok::<_, DataFusionError>(PhysicalSortExpr {
760+
expr: ctx.decode_expr(expr_node, schema.as_ref())?,
761+
options: arrow::compute::SortOptions {
762+
descending: !sort_expr.asc,
763+
nulls_first: sort_expr.nulls_first,
764+
},
765+
})
766+
})
767+
.collect::<Result<Vec<_>>>()?;
768+
sort_information.extend(LexOrdering::new(sort_exprs));
769+
}
770+
771+
let source = Self::try_new(&partitions, schema, projection)?
772+
.with_limit(scan.fetch.map(|f| f as usize))
773+
.with_show_sizes(scan.show_sizes);
774+
let source = source.try_with_sort_information(sort_information)?;
775+
776+
Ok(DataSourceExec::from_data_source(source))
777+
}
778+
}
779+
610780
/// For use in repartitioning, track the total size and original partition index.
611781
///
612782
/// Do not implement clone, in order to avoid unnecessary copying during repartitioning.

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 4 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ use datafusion_common::{
3434
use datafusion_datasource::file_compression_type::FileCompressionType;
3535
use datafusion_datasource::file_scan_config::FileScanConfigBuilder;
3636
use datafusion_datasource::sink::DataSinkExec;
37-
use datafusion_datasource::source::{DataSource, DataSourceExec};
37+
use datafusion_datasource::source::DataSourceExec;
3838
use datafusion_datasource_arrow::source::ArrowSource;
3939
#[cfg(feature = "avro")]
4040
use datafusion_datasource_avro::source::AvroSource;
@@ -111,8 +111,7 @@ use crate::physical_plan::from_proto::{
111111
};
112112
use crate::physical_plan::to_proto::{
113113
serialize_maybe_filter, serialize_physical_aggr_expr,
114-
serialize_physical_expr_with_converter, serialize_physical_sort_exprs,
115-
serialize_physical_window_expr, serialize_record_batches,
114+
serialize_physical_expr_with_converter, serialize_physical_window_expr,
116115
};
117116
use crate::protobuf::physical_aggregate_expr_node::AggregateFunction;
118117
use crate::protobuf::physical_expr_node::ExprType;
@@ -781,8 +780,8 @@ pub trait PhysicalPlanNodeExt: Sized {
781780
"Unable to process a Avro PhysicalPlan when `avro` feature is not enabled"
782781
)
783782
}
784-
PhysicalPlanType::MemoryScan(scan) => {
785-
self.try_into_memory_scan_physical_plan(scan, ctx, proto_converter)
783+
PhysicalPlanType::MemoryScan(_) => {
784+
MemorySourceConfig::try_from_proto(self.node(), &decode_ctx)
786785
}
787786
PhysicalPlanType::ArrowScan(_) => {
788787
ArrowSource::try_from_proto(self.node(), &decode_ctx)
@@ -914,16 +913,6 @@ pub trait PhysicalPlanNodeExt: Sized {
914913
return Ok(node);
915914
}
916915

917-
if let Some(data_source_exec) = plan.downcast_ref::<DataSourceExec>()
918-
&& let Some(node) = protobuf::PhysicalPlanNode::try_from_data_source_exec(
919-
data_source_exec,
920-
codec,
921-
proto_converter,
922-
)?
923-
{
924-
return Ok(node);
925-
}
926-
927916
if let Some(exec) = plan.downcast_ref::<SortPreservingMergeExec>() {
928917
return protobuf::PhysicalPlanNode::try_from_sort_preserving_merge_exec(
929918
exec,
@@ -3337,67 +3326,6 @@ pub trait PhysicalPlanNodeExt: Sized {
33373326
))),
33383327
})
33393328
}
3340-
3341-
fn try_from_data_source_exec(
3342-
data_source_exec: &DataSourceExec,
3343-
codec: &dyn PhysicalExtensionCodec,
3344-
proto_converter: &dyn PhysicalProtoConverterExtension,
3345-
) -> Result<Option<protobuf::PhysicalPlanNode>> {
3346-
let data_source = data_source_exec.data_source();
3347-
// Csv/Json/Arrow/Avro/Parquet file sources serialize themselves via the
3348-
// `FileSource::try_to_proto` hook (#22419); their inline branches are
3349-
// deleted to prove the hook is reached. Only MemorySourceConfig (not a
3350-
// FileSource) remains here.
3351-
3352-
if let Some(source_conf) = data_source.downcast_ref::<MemorySourceConfig>() {
3353-
let proto_partitions = source_conf
3354-
.partitions()
3355-
.iter()
3356-
.map(|p| serialize_record_batches(p))
3357-
.collect::<Result<Vec<_>>>()?;
3358-
3359-
let proto_schema: protobuf::Schema =
3360-
source_conf.original_schema().as_ref().try_into()?;
3361-
3362-
let proto_projection = source_conf
3363-
.projection()
3364-
.as_ref()
3365-
.map_or_else(Vec::new, |v| {
3366-
v.iter().map(|x| *x as u32).collect::<Vec<u32>>()
3367-
});
3368-
3369-
let proto_sort_information = source_conf
3370-
.sort_information()
3371-
.iter()
3372-
.map(|ordering| {
3373-
let sort_exprs = serialize_physical_sort_exprs(
3374-
ordering.to_owned(),
3375-
codec,
3376-
proto_converter,
3377-
)?;
3378-
Ok::<_, DataFusionError>(protobuf::PhysicalSortExprNodeCollection {
3379-
physical_sort_expr_nodes: sort_exprs,
3380-
})
3381-
})
3382-
.collect::<Result<Vec<_>, _>>()?;
3383-
3384-
return Ok(Some(protobuf::PhysicalPlanNode {
3385-
physical_plan_type: Some(PhysicalPlanType::MemoryScan(
3386-
protobuf::MemoryScanExecNode {
3387-
partitions: proto_partitions,
3388-
schema: Some(proto_schema),
3389-
projection: proto_projection,
3390-
sort_information: proto_sort_information,
3391-
show_sizes: source_conf.show_sizes(),
3392-
fetch: source_conf.fetch().map(|f| f as u32),
3393-
},
3394-
)),
3395-
}));
3396-
}
3397-
3398-
Ok(None)
3399-
}
3400-
34013329
fn try_from_coalesce_partitions_exec(
34023330
exec: &CoalescePartitionsExec,
34033331
codec: &dyn PhysicalExtensionCodec,

0 commit comments

Comments
 (0)