Skip to content

Commit 62da8a5

Browse files
committed
refactor(proto): migrate CsvSink serde
CsvSink serialization still depended on central downcasts in datafusion-proto. Moving encode and decode ownership into the CSV sink exercises the DataSink hook and keeps format-specific wire logic with the concrete type. Retain the old decode helper as a deprecated compatibility shim. Refs apache#23519
1 parent 99a7f62 commit 62da8a5

6 files changed

Lines changed: 145 additions & 46 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/datasource-csv/Cargo.toml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,13 @@ version.workspace = true
3030
[package.metadata.docs.rs]
3131
all-features = true
3232

33+
[features]
34+
proto = [
35+
"dep:datafusion-proto-models",
36+
"datafusion-datasource/proto",
37+
"datafusion-physical-plan/proto",
38+
]
39+
3340
[dependencies]
3441
arrow = { workspace = true }
3542
async-trait = { workspace = true }
@@ -41,6 +48,7 @@ datafusion-execution = { workspace = true }
4148
datafusion-expr = { workspace = true }
4249
datafusion-physical-expr-common = { workspace = true }
4350
datafusion-physical-plan = { workspace = true }
51+
datafusion-proto-models = { workspace = true, optional = true }
4452
datafusion-session = { workspace = true }
4553
futures = { workspace = true }
4654
object_store = { workspace = true }

‎datafusion/datasource-csv/src/file_format.rs‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -825,6 +825,91 @@ impl DataSink for CsvSink {
825825
) -> Result<u64> {
826826
FileSink::write_all(self, data, context).await
827827
}
828+
829+
#[cfg(feature = "proto")]
830+
fn try_to_proto(
831+
&self,
832+
exec: &DataSinkExec,
833+
ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
834+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
835+
use datafusion_proto_models::protobuf;
836+
use protobuf::physical_plan_node::PhysicalPlanType;
837+
838+
let input = ctx.encode_child(exec.input())?;
839+
let sort_order = exec.encode_sort_order(ctx)?;
840+
let sink = protobuf::CsvSink {
841+
config: Some((&self.config).try_into()?),
842+
writer_options: Some(self.writer_options().try_into()?),
843+
};
844+
let node = protobuf::CsvSinkExecNode {
845+
input: Some(Box::new(input)),
846+
sink: Some(sink),
847+
sink_schema: Some(exec.schema().as_ref().try_into()?),
848+
sort_order,
849+
};
850+
Ok(Some(protobuf::PhysicalPlanNode {
851+
physical_plan_type: Some(PhysicalPlanType::CsvSink(Box::new(node))),
852+
}))
853+
}
854+
}
855+
856+
#[cfg(feature = "proto")]
857+
impl CsvSink {
858+
/// Reconstructs a [`DataSinkExec`] containing a `CsvSink` from protobuf.
859+
pub fn try_from_proto(
860+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
861+
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
862+
) -> Result<Arc<dyn ExecutionPlan>> {
863+
use datafusion_proto_models::protobuf;
864+
865+
let sink_node = match &node.physical_plan_type {
866+
Some(protobuf::physical_plan_node::PhysicalPlanType::CsvSink(sink)) => {
867+
sink.as_ref()
868+
}
869+
_ => {
870+
return datafusion_common::internal_err!(
871+
"PhysicalPlanNode is not a CsvSink"
872+
);
873+
}
874+
};
875+
let input = ctx.decode_required_child(
876+
sink_node.input.as_deref(),
877+
"CsvSinkExecNode",
878+
"input",
879+
)?;
880+
let proto_sink = sink_node.sink.as_ref().ok_or_else(|| {
881+
datafusion_common::internal_datafusion_err!(
882+
"CsvSinkExecNode is missing required field 'sink'"
883+
)
884+
})?;
885+
let config =
886+
FileSinkConfig::try_from(proto_sink.config.as_ref().ok_or_else(|| {
887+
datafusion_common::internal_datafusion_err!(
888+
"CsvSink is missing required field 'config'"
889+
)
890+
})?)?;
891+
let writer_options = proto_sink
892+
.writer_options
893+
.as_ref()
894+
.ok_or_else(|| {
895+
datafusion_common::internal_datafusion_err!(
896+
"CsvSink is missing required field 'writer_options'"
897+
)
898+
})?
899+
.try_into()?;
900+
let data_sink = CsvSink::new(config, writer_options);
901+
let sort_order = DataSinkExec::decode_sort_order(
902+
sink_node.sort_order.as_ref(),
903+
ctx,
904+
input.schema().as_ref(),
905+
)?;
906+
907+
Ok(Arc::new(DataSinkExec::new(
908+
input,
909+
Arc::new(data_sink),
910+
sort_order,
911+
)))
912+
}
828913
}
829914

830915
#[cfg(test)]

‎datafusion/datasource/src/sink.rs‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ use std::sync::Arc;
2424

2525
use arrow::array::{ArrayRef, RecordBatch, UInt64Array};
2626
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
27-
use datafusion_common::{Result, assert_eq_or_internal_err};
27+
use datafusion_common::{Result, assert_eq_or_internal_err, internal_datafusion_err};
2828
use datafusion_execution::TaskContext;
2929
use datafusion_physical_expr::{Distribution, EquivalenceProperties};
3030
use datafusion_physical_expr_common::sort_expr::{LexRequirement, OrderingRequirements};
@@ -194,6 +194,40 @@ impl DataSinkExec {
194194
.transpose()
195195
}
196196

197+
/// Decode the optional sink ordering from a protobuf plan node.
198+
#[cfg(feature = "proto")]
199+
pub fn decode_sort_order(
200+
collection: Option<
201+
&datafusion_proto_models::protobuf::PhysicalSortExprNodeCollection,
202+
>,
203+
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
204+
schema: &Schema,
205+
) -> Result<Option<LexRequirement>> {
206+
use arrow::compute::SortOptions;
207+
use datafusion_physical_expr::PhysicalSortExpr;
208+
209+
let Some(collection) = collection else {
210+
return Ok(None);
211+
};
212+
let sort_exprs = collection
213+
.physical_sort_expr_nodes
214+
.iter()
215+
.map(|node| {
216+
let expr = node.expr.as_ref().ok_or_else(|| {
217+
internal_datafusion_err!("Unexpected empty physical expression")
218+
})?;
219+
Ok(PhysicalSortExpr {
220+
expr: ctx.decode_expr(expr, schema)?,
221+
options: SortOptions {
222+
descending: !node.asc,
223+
nulls_first: node.nulls_first,
224+
},
225+
})
226+
})
227+
.collect::<Result<Vec<_>>>()?;
228+
Ok(LexRequirement::new(sort_exprs.into_iter().map(Into::into)))
229+
}
230+
197231
fn create_schema(
198232
input: &Arc<dyn ExecutionPlan>,
199233
schema: SchemaRef,

‎datafusion/proto/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ datafusion-common = { workspace = true }
5757
datafusion-datasource = { workspace = true, features = ["proto"] }
5858
datafusion-datasource-arrow = { workspace = true }
5959
datafusion-datasource-avro = { workspace = true, optional = true }
60-
datafusion-datasource-csv = { workspace = true }
60+
datafusion-datasource-csv = { workspace = true, features = ["proto"] }
6161
datafusion-datasource-json = { workspace = true }
6262
datafusion-datasource-parquet = { workspace = true, optional = true }
6363
datafusion-execution = { workspace = true }

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 15 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -786,8 +786,8 @@ pub trait PhysicalPlanNodeExt: Sized {
786786
PhysicalPlanType::JsonSink(sink) => {
787787
self.try_into_json_sink_physical_plan(sink, ctx, proto_converter)
788788
}
789-
PhysicalPlanType::CsvSink(sink) => {
790-
self.try_into_csv_sink_physical_plan(sink, ctx, proto_converter)
789+
PhysicalPlanType::CsvSink(_) => {
790+
CsvSink::try_from_proto(self.node(), &decode_ctx)
791791
}
792792
#[cfg_attr(not(feature = "parquet"), allow(unused_variables))]
793793
PhysicalPlanType::ParquetSink(sink) => {
@@ -1672,41 +1672,25 @@ pub trait PhysicalPlanNodeExt: Sized {
16721672
)))
16731673
}
16741674

1675+
#[deprecated(
1676+
since = "55.0.0",
1677+
note = "unused by DataFusion; `CsvSink` deserializes itself via `CsvSink::try_from_proto`"
1678+
)]
16751679
fn try_into_csv_sink_physical_plan(
16761680
&self,
16771681
sink: &protobuf::CsvSinkExecNode,
16781682
ctx: &PhysicalPlanDecodeContext<'_>,
16791683
proto_converter: &dyn PhysicalProtoConverterExtension,
16801684
) -> Result<Arc<dyn ExecutionPlan>> {
1681-
let input = into_physical_plan(&sink.input, ctx, proto_converter)?;
1682-
1683-
let data_sink = CsvSink::try_from_proto(
1684-
sink.sink
1685-
.as_ref()
1686-
.ok_or_else(|| proto_error("Missing required field in protobuf"))?,
1687-
)?;
1688-
let sink_schema = input.schema();
1689-
let sort_order = sink
1690-
.sort_order
1691-
.as_ref()
1692-
.map(|collection| {
1693-
parse_physical_sort_exprs(
1694-
&collection.physical_sort_expr_nodes,
1695-
ctx,
1696-
&sink_schema,
1697-
proto_converter,
1698-
)
1699-
.map(|sort_exprs| {
1700-
LexRequirement::new(sort_exprs.into_iter().map(Into::into))
1701-
})
1702-
})
1703-
.transpose()?
1704-
.flatten();
1705-
Ok(Arc::new(DataSinkExec::new(
1706-
input,
1707-
Arc::new(data_sink),
1708-
sort_order,
1709-
)))
1685+
let node = protobuf::PhysicalPlanNode {
1686+
physical_plan_type: Some(PhysicalPlanType::CsvSink(Box::new(sink.clone()))),
1687+
};
1688+
let decoder = ConverterPlanDecoder {
1689+
ctx,
1690+
proto_converter,
1691+
};
1692+
let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
1693+
CsvSink::try_from_proto(&node, &decode_ctx)
17101694
}
17111695

17121696
#[cfg_attr(not(feature = "parquet"), expect(unused_variables))]
@@ -2620,19 +2604,6 @@ pub trait PhysicalPlanNodeExt: Sized {
26202604
}));
26212605
}
26222606

2623-
if let Some(sink) = exec.sink().downcast_ref::<CsvSink>() {
2624-
return Ok(Some(protobuf::PhysicalPlanNode {
2625-
physical_plan_type: Some(PhysicalPlanType::CsvSink(Box::new(
2626-
protobuf::CsvSinkExecNode {
2627-
input: Some(Box::new(input)),
2628-
sink: Some(protobuf::CsvSink::try_from_proto(sink)?),
2629-
sink_schema: Some(exec.schema().as_ref().try_into()?),
2630-
sort_order,
2631-
},
2632-
))),
2633-
}));
2634-
}
2635-
26362607
#[cfg(feature = "parquet")]
26372608
if let Some(sink) = exec.sink().downcast_ref::<ParquetSink>() {
26382609
return Ok(Some(protobuf::PhysicalPlanNode {

0 commit comments

Comments
 (0)