Skip to content

Commit 12b13d9

Browse files
committed
refactor(proto): migrate JsonSink serde
JsonSink serialization still depended on central downcasts in datafusion-proto. Moving encode and decode ownership into the JSON sink exercises the DataSink hook and keeps format-specific wire logic with the concrete type. Retain the old decode helper as a deprecated compatibility shim. Refs apache#23519 Signed-off-by: Jiawei Zhao <[email protected]>
1 parent 5aa03bf commit 12b13d9

5 files changed

Lines changed: 110 additions & 45 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/datasource-json/Cargo.toml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,13 @@ version.workspace = true
3030
[package.metadata.docs.rs]
3131
all-features = true
3232

33+
[features]
34+
proto = [
35+
"dep:datafusion-proto-models",
36+
"datafusion-datasource/proto",
37+
"datafusion-physical-plan/proto",
38+
]
39+
3340
[dependencies]
3441
arrow = { workspace = true }
3542
async-trait = { workspace = true }
@@ -41,6 +48,7 @@ datafusion-execution = { workspace = true }
4148
datafusion-expr = { workspace = true }
4249
datafusion-physical-expr-common = { workspace = true }
4350
datafusion-physical-plan = { workspace = true }
51+
datafusion-proto-models = { workspace = true, optional = true }
4452
datafusion-session = { workspace = true }
4553
futures = { workspace = true }
4654
object_store = { workspace = true }

‎datafusion/datasource-json/src/file_format.rs‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -490,6 +490,91 @@ impl DataSink for JsonSink {
490490
) -> Result<u64> {
491491
FileSink::write_all(self, data, context).await
492492
}
493+
494+
#[cfg(feature = "proto")]
495+
fn try_to_proto(
496+
&self,
497+
exec: &DataSinkExec,
498+
ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
499+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
500+
use datafusion_proto_models::protobuf;
501+
use protobuf::physical_plan_node::PhysicalPlanType;
502+
503+
let input = ctx.encode_child(exec.input())?;
504+
let sort_order = exec.encode_sort_order(ctx)?;
505+
let sink = protobuf::JsonSink {
506+
config: Some((&self.config).try_into()?),
507+
writer_options: Some(self.writer_options().try_into()?),
508+
};
509+
let node = protobuf::JsonSinkExecNode {
510+
input: Some(Box::new(input)),
511+
sink: Some(sink),
512+
sink_schema: Some(exec.schema().as_ref().try_into()?),
513+
sort_order,
514+
};
515+
Ok(Some(protobuf::PhysicalPlanNode {
516+
physical_plan_type: Some(PhysicalPlanType::JsonSink(Box::new(node))),
517+
}))
518+
}
519+
}
520+
521+
#[cfg(feature = "proto")]
522+
impl JsonSink {
523+
/// Reconstructs a [`DataSinkExec`] containing a `JsonSink` from protobuf.
524+
pub fn try_from_proto(
525+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
526+
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
527+
) -> Result<Arc<dyn ExecutionPlan>> {
528+
use datafusion_proto_models::protobuf;
529+
530+
let sink_node = match &node.physical_plan_type {
531+
Some(protobuf::physical_plan_node::PhysicalPlanType::JsonSink(sink)) => {
532+
sink.as_ref()
533+
}
534+
_ => {
535+
return datafusion_common::internal_err!(
536+
"PhysicalPlanNode is not a JsonSink"
537+
);
538+
}
539+
};
540+
let input = ctx.decode_required_child(
541+
sink_node.input.as_deref(),
542+
"JsonSinkExecNode",
543+
"input",
544+
)?;
545+
let proto_sink = sink_node.sink.as_ref().ok_or_else(|| {
546+
datafusion_common::internal_datafusion_err!(
547+
"JsonSinkExecNode is missing required field 'sink'"
548+
)
549+
})?;
550+
let config =
551+
FileSinkConfig::try_from(proto_sink.config.as_ref().ok_or_else(|| {
552+
datafusion_common::internal_datafusion_err!(
553+
"JsonSink is missing required field 'config'"
554+
)
555+
})?)?;
556+
let writer_options = proto_sink
557+
.writer_options
558+
.as_ref()
559+
.ok_or_else(|| {
560+
datafusion_common::internal_datafusion_err!(
561+
"JsonSink is missing required field 'writer_options'"
562+
)
563+
})?
564+
.try_into()?;
565+
let data_sink = JsonSink::new(config, writer_options);
566+
let sort_order = DataSinkExec::decode_sort_order(
567+
sink_node.sort_order.as_ref(),
568+
ctx,
569+
input.schema().as_ref(),
570+
)?;
571+
572+
Ok(Arc::new(DataSinkExec::new(
573+
input,
574+
Arc::new(data_sink),
575+
sort_order,
576+
)))
577+
}
493578
}
494579

495580
#[derive(Debug)]

‎datafusion/proto/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ datafusion-datasource = { workspace = true, features = ["proto"] }
6464
datafusion-datasource-arrow = { workspace = true }
6565
datafusion-datasource-avro = { workspace = true, optional = true }
6666
datafusion-datasource-csv = { workspace = true, features = ["proto"] }
67-
datafusion-datasource-json = { workspace = true }
67+
datafusion-datasource-json = { workspace = true, features = ["proto"] }
6868
datafusion-datasource-parquet = { workspace = true, optional = true }
6969
datafusion-execution = { workspace = true }
7070
datafusion-expr = { workspace = true }

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 15 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -782,8 +782,8 @@ pub trait PhysicalPlanNodeExt: Sized {
782782
PhysicalPlanType::Analyze(_) => {
783783
AnalyzeExec::try_from_proto(self.node(), &decode_ctx)
784784
}
785-
PhysicalPlanType::JsonSink(sink) => {
786-
self.try_into_json_sink_physical_plan(sink, ctx, proto_converter)
785+
PhysicalPlanType::JsonSink(_) => {
786+
JsonSink::try_from_proto(self.node(), &decode_ctx)
787787
}
788788
PhysicalPlanType::CsvSink(_) => {
789789
CsvSink::try_from_proto(self.node(), &decode_ctx)
@@ -1630,41 +1630,25 @@ pub trait PhysicalPlanNodeExt: Sized {
16301630
AnalyzeExec::try_from_proto(self.node(), &decode_ctx)
16311631
}
16321632

1633+
#[deprecated(
1634+
since = "55.0.0",
1635+
note = "unused by DataFusion; `JsonSink` deserializes itself via `JsonSink::try_from_proto`"
1636+
)]
16331637
fn try_into_json_sink_physical_plan(
16341638
&self,
16351639
sink: &protobuf::JsonSinkExecNode,
16361640
ctx: &PhysicalPlanDecodeContext<'_>,
16371641
proto_converter: &dyn PhysicalProtoConverterExtension,
16381642
) -> Result<Arc<dyn ExecutionPlan>> {
1639-
let input = into_physical_plan(&sink.input, ctx, proto_converter)?;
1640-
1641-
let data_sink = JsonSink::try_from_proto(
1642-
sink.sink
1643-
.as_ref()
1644-
.ok_or_else(|| proto_error("Missing required field in protobuf"))?,
1645-
)?;
1646-
let sink_schema = input.schema();
1647-
let sort_order = sink
1648-
.sort_order
1649-
.as_ref()
1650-
.map(|collection| {
1651-
parse_physical_sort_exprs(
1652-
&collection.physical_sort_expr_nodes,
1653-
ctx,
1654-
&sink_schema,
1655-
proto_converter,
1656-
)
1657-
.map(|sort_exprs| {
1658-
LexRequirement::new(sort_exprs.into_iter().map(Into::into))
1659-
})
1660-
})
1661-
.transpose()?
1662-
.flatten();
1663-
Ok(Arc::new(DataSinkExec::new(
1664-
input,
1665-
Arc::new(data_sink),
1666-
sort_order,
1667-
)))
1643+
let node = protobuf::PhysicalPlanNode {
1644+
physical_plan_type: Some(PhysicalPlanType::JsonSink(Box::new(sink.clone()))),
1645+
};
1646+
let decoder = ConverterPlanDecoder {
1647+
ctx,
1648+
proto_converter,
1649+
};
1650+
let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
1651+
JsonSink::try_from_proto(&node, &decode_ctx)
16681652
}
16691653

16701654
#[deprecated(
@@ -2570,19 +2554,6 @@ pub trait PhysicalPlanNodeExt: Sized {
25702554
let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder);
25712555
let sort_order = exec.encode_sort_order(&encode_ctx)?;
25722556

2573-
if let Some(sink) = exec.sink().downcast_ref::<JsonSink>() {
2574-
return Ok(Some(protobuf::PhysicalPlanNode {
2575-
physical_plan_type: Some(PhysicalPlanType::JsonSink(Box::new(
2576-
protobuf::JsonSinkExecNode {
2577-
input: Some(Box::new(input)),
2578-
sink: Some(protobuf::JsonSink::try_from_proto(sink)?),
2579-
sink_schema: Some(exec.schema().as_ref().try_into()?),
2580-
sort_order,
2581-
},
2582-
))),
2583-
}));
2584-
}
2585-
25862557
#[cfg(feature = "parquet")]
25872558
if let Some(sink) = exec.sink().downcast_ref::<ParquetSink>() {
25882559
return Ok(Some(protobuf::PhysicalPlanNode {

0 commit comments

Comments
 (0)