Skip to content

Commit b8d4f6d

Browse files
adriangbclaude
andcommitted
prototype: migrate the sink family (DataSink hook + 3 file sinks) onto the proto hook (apache#22419)
Mirrors the scan side: DataSink::try_to_proto hook on the trait, DataSinkExec rides the ExecutionPlan hook and delegates to its sink (encoding input + sort_order via the ctx, since a sink carries no child exprs). FileSinkConfig serde ported into datafusion-datasource (file_sink_config/proto.rs), reusing the scan side's PartitionedFile helpers; no codec needed. CsvSink/JsonSink/ParquetSink each implement DataSink::try_to_proto + an inherent try_from_proto. The central try_from_data_sink_exec, the DataSinkExec encode arm, the try_into_*_sink_physical_plan decoders, and the colliding TryFromProto<&protobuf::*Sink> for *Sink decode impls are all deleted; the 3 sink decode arms repoint to <Fmt>Sink::try_from_proto. roundtrip_{csv,json,parquet}_sink stay green (193 tests). Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01SfBPC2yYCwHUn4V8pQCVj3
1 parent 853b07c commit b8d4f6d

10 files changed

Lines changed: 607 additions & 259 deletions

File tree

‎datafusion/datasource-csv/src/file_format.rs‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -826,6 +826,104 @@ impl DataSink for CsvSink {
826826
) -> Result<u64> {
827827
FileSink::write_all(self, data, context).await
828828
}
829+
830+
/// Emit a `CsvSink` node wrapping the shared sink config. Kept
831+
/// byte-identical to the former `try_from_data_sink_exec` CsvSink branch in
832+
/// `datafusion-proto`.
833+
#[cfg(feature = "proto")]
834+
fn try_to_proto(
835+
&self,
836+
input: datafusion_proto_models::protobuf::PhysicalPlanNode,
837+
sort_order: Option<
838+
datafusion_proto_models::protobuf::PhysicalSortExprNodeCollection,
839+
>,
840+
sink_schema: &Schema,
841+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
842+
use datafusion_proto_models::protobuf;
843+
use protobuf::physical_plan_node::PhysicalPlanType;
844+
845+
let sink = protobuf::CsvSink {
846+
config: Some(self.config.to_proto()?),
847+
writer_options: Some(self.writer_options().try_into()?),
848+
};
849+
let node = protobuf::CsvSinkExecNode {
850+
input: Some(Box::new(input)),
851+
sink: Some(sink),
852+
sink_schema: Some(sink_schema.try_into()?),
853+
sort_order,
854+
};
855+
Ok(Some(protobuf::PhysicalPlanNode {
856+
physical_plan_type: Some(PhysicalPlanType::CsvSink(Box::new(node))),
857+
}))
858+
}
859+
}
860+
861+
#[cfg(feature = "proto")]
862+
impl CsvSink {
863+
/// Reconstruct a `DataSinkExec` (wrapping a `CsvSink`) from a `CsvSink`
864+
/// [`PhysicalPlanNode`].
865+
///
866+
/// The inverse of [`DataSink::try_to_proto`] on `CsvSink`; kept
867+
/// byte-compatible with the former `try_into_csv_sink_physical_plan` in
868+
/// `datafusion-proto`.
869+
///
870+
/// [`PhysicalPlanNode`]: datafusion_proto_models::protobuf::PhysicalPlanNode
871+
pub fn try_from_proto(
872+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
873+
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
874+
) -> Result<Arc<dyn ExecutionPlan>> {
875+
use datafusion_datasource::file_sink_config::parse_sink_sort_order;
876+
use datafusion_proto_models::protobuf;
877+
878+
let sink_node = match &node.physical_plan_type {
879+
Some(protobuf::physical_plan_node::PhysicalPlanType::CsvSink(sink)) => {
880+
sink.as_ref()
881+
}
882+
_ => {
883+
return datafusion_common::internal_err!(
884+
"PhysicalPlanNode is not a CsvSink"
885+
);
886+
}
887+
};
888+
889+
let input = ctx.decode_required_child(
890+
sink_node.input.as_deref(),
891+
"CsvSinkExecNode",
892+
"input",
893+
)?;
894+
895+
let proto_sink = sink_node.sink.as_ref().ok_or_else(|| {
896+
datafusion_common::internal_datafusion_err!(
897+
"CsvSinkExecNode is missing required field 'sink'"
898+
)
899+
})?;
900+
let config =
901+
FileSinkConfig::from_proto(proto_sink.config.as_ref().ok_or_else(|| {
902+
datafusion_common::internal_datafusion_err!(
903+
"CsvSink is missing required field 'config'"
904+
)
905+
})?)?;
906+
let writer_options = proto_sink
907+
.writer_options
908+
.as_ref()
909+
.ok_or_else(|| {
910+
datafusion_common::internal_datafusion_err!(
911+
"CsvSink is missing required field 'writer_options'"
912+
)
913+
})?
914+
.try_into()?;
915+
let data_sink = CsvSink::new(config, writer_options);
916+
917+
let sink_schema = input.schema();
918+
let sort_order =
919+
parse_sink_sort_order(sink_node.sort_order.as_ref(), ctx, &sink_schema)?;
920+
921+
Ok(Arc::new(DataSinkExec::new(
922+
input,
923+
Arc::new(data_sink),
924+
sort_order,
925+
)))
926+
}
829927
}
830928

831929
#[cfg(test)]

‎datafusion/datasource-json/src/file_format.rs‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -490,6 +490,105 @@ impl DataSink for JsonSink {
490490
) -> Result<u64> {
491491
FileSink::write_all(self, data, context).await
492492
}
493+
494+
/// Emit a `JsonSink` node wrapping the shared sink config. Kept
495+
/// byte-identical to the former `try_from_data_sink_exec` JsonSink branch in
496+
/// `datafusion-proto`.
497+
#[cfg(feature = "proto")]
498+
fn try_to_proto(
499+
&self,
500+
input: datafusion_proto_models::protobuf::PhysicalPlanNode,
501+
sort_order: Option<
502+
datafusion_proto_models::protobuf::PhysicalSortExprNodeCollection,
503+
>,
504+
sink_schema: &Schema,
505+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
506+
use datafusion_proto_models::protobuf;
507+
use protobuf::physical_plan_node::PhysicalPlanType;
508+
509+
let sink = protobuf::JsonSink {
510+
config: Some(self.config.to_proto()?),
511+
writer_options: Some(self.writer_options().try_into()?),
512+
};
513+
let node = protobuf::JsonSinkExecNode {
514+
input: Some(Box::new(input)),
515+
sink: Some(sink),
516+
sink_schema: Some(sink_schema.try_into()?),
517+
sort_order,
518+
};
519+
Ok(Some(protobuf::PhysicalPlanNode {
520+
physical_plan_type: Some(PhysicalPlanType::JsonSink(Box::new(node))),
521+
}))
522+
}
523+
}
524+
525+
#[cfg(feature = "proto")]
526+
impl JsonSink {
527+
/// Reconstruct a `DataSinkExec` (wrapping a `JsonSink`) from a `JsonSink`
528+
/// [`PhysicalPlanNode`].
529+
///
530+
/// The inverse of [`DataSink::try_to_proto`] on `JsonSink`; kept
531+
/// byte-compatible with the former `try_into_json_sink_physical_plan` in
532+
/// `datafusion-proto`.
533+
///
534+
/// [`PhysicalPlanNode`]: datafusion_proto_models::protobuf::PhysicalPlanNode
535+
pub fn try_from_proto(
536+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
537+
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
538+
) -> Result<Arc<dyn ExecutionPlan>> {
539+
use datafusion_datasource::file_sink_config::parse_sink_sort_order;
540+
use datafusion_datasource::sink::DataSinkExec;
541+
use datafusion_proto_models::protobuf;
542+
543+
let sink_node = match &node.physical_plan_type {
544+
Some(protobuf::physical_plan_node::PhysicalPlanType::JsonSink(sink)) => {
545+
sink.as_ref()
546+
}
547+
_ => {
548+
return datafusion_common::internal_err!(
549+
"PhysicalPlanNode is not a JsonSink"
550+
);
551+
}
552+
};
553+
554+
let input = ctx.decode_required_child(
555+
sink_node.input.as_deref(),
556+
"JsonSinkExecNode",
557+
"input",
558+
)?;
559+
560+
let proto_sink = sink_node.sink.as_ref().ok_or_else(|| {
561+
datafusion_common::internal_datafusion_err!(
562+
"JsonSinkExecNode is missing required field 'sink'"
563+
)
564+
})?;
565+
let config =
566+
FileSinkConfig::from_proto(proto_sink.config.as_ref().ok_or_else(|| {
567+
datafusion_common::internal_datafusion_err!(
568+
"JsonSink is missing required field 'config'"
569+
)
570+
})?)?;
571+
let writer_options = proto_sink
572+
.writer_options
573+
.as_ref()
574+
.ok_or_else(|| {
575+
datafusion_common::internal_datafusion_err!(
576+
"JsonSink is missing required field 'writer_options'"
577+
)
578+
})?
579+
.try_into()?;
580+
let data_sink = JsonSink::new(config, writer_options);
581+
582+
let sink_schema = input.schema();
583+
let sort_order =
584+
parse_sink_sort_order(sink_node.sort_order.as_ref(), ctx, &sink_schema)?;
585+
586+
Ok(Arc::new(DataSinkExec::new(
587+
input,
588+
Arc::new(data_sink),
589+
sort_order,
590+
)))
591+
}
493592
}
494593

495594
#[derive(Debug)]

‎datafusion/datasource-parquet/src/sink.rs‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ use datafusion_datasource::write::{
4040
use datafusion_execution::memory_pool::{MemoryConsumer, MemoryPool, MemoryReservation};
4141
use datafusion_execution::runtime_env::RuntimeEnv;
4242
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
43+
#[cfg(feature = "proto")]
44+
use datafusion_physical_plan::ExecutionPlan;
4345
use datafusion_physical_plan::metrics::{
4446
ElapsedComputeFutureExt, ExecutionPlanMetricsSet, MetricBuilder, MetricCategory,
4547
MetricsSet, Time,
@@ -411,6 +413,105 @@ impl DataSink for ParquetSink {
411413
) -> Result<u64> {
412414
FileSink::write_all(self, data, context).await
413415
}
416+
417+
/// Emit a `ParquetSink` node wrapping the shared sink config. Kept
418+
/// byte-identical to the former `try_from_data_sink_exec` ParquetSink branch
419+
/// in `datafusion-proto`.
420+
#[cfg(feature = "proto")]
421+
fn try_to_proto(
422+
&self,
423+
input: datafusion_proto_models::protobuf::PhysicalPlanNode,
424+
sort_order: Option<
425+
datafusion_proto_models::protobuf::PhysicalSortExprNodeCollection,
426+
>,
427+
sink_schema: &Schema,
428+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
429+
use datafusion_proto_models::protobuf;
430+
use protobuf::physical_plan_node::PhysicalPlanType;
431+
432+
let sink = protobuf::ParquetSink {
433+
config: Some(self.config.to_proto()?),
434+
parquet_options: Some(self.parquet_options().try_into()?),
435+
};
436+
let node = protobuf::ParquetSinkExecNode {
437+
input: Some(Box::new(input)),
438+
sink: Some(sink),
439+
sink_schema: Some(sink_schema.try_into()?),
440+
sort_order,
441+
};
442+
Ok(Some(protobuf::PhysicalPlanNode {
443+
physical_plan_type: Some(PhysicalPlanType::ParquetSink(Box::new(node))),
444+
}))
445+
}
446+
}
447+
448+
#[cfg(feature = "proto")]
449+
impl ParquetSink {
450+
/// Reconstruct a `DataSinkExec` (wrapping a `ParquetSink`) from a
451+
/// `ParquetSink` [`PhysicalPlanNode`].
452+
///
453+
/// The inverse of [`DataSink::try_to_proto`] on `ParquetSink`; kept
454+
/// byte-compatible with the former `try_into_parquet_sink_physical_plan` in
455+
/// `datafusion-proto`.
456+
///
457+
/// [`PhysicalPlanNode`]: datafusion_proto_models::protobuf::PhysicalPlanNode
458+
pub fn try_from_proto(
459+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
460+
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
461+
) -> Result<Arc<dyn ExecutionPlan>> {
462+
use datafusion_datasource::file_sink_config::parse_sink_sort_order;
463+
use datafusion_datasource::sink::DataSinkExec;
464+
use datafusion_proto_models::protobuf;
465+
466+
let sink_node = match &node.physical_plan_type {
467+
Some(protobuf::physical_plan_node::PhysicalPlanType::ParquetSink(sink)) => {
468+
sink.as_ref()
469+
}
470+
_ => {
471+
return datafusion_common::internal_err!(
472+
"PhysicalPlanNode is not a ParquetSink"
473+
);
474+
}
475+
};
476+
477+
let input = ctx.decode_required_child(
478+
sink_node.input.as_deref(),
479+
"ParquetSinkExecNode",
480+
"input",
481+
)?;
482+
483+
let proto_sink = sink_node.sink.as_ref().ok_or_else(|| {
484+
datafusion_common::internal_datafusion_err!(
485+
"ParquetSinkExecNode is missing required field 'sink'"
486+
)
487+
})?;
488+
let config =
489+
FileSinkConfig::from_proto(proto_sink.config.as_ref().ok_or_else(|| {
490+
datafusion_common::internal_datafusion_err!(
491+
"ParquetSink is missing required field 'config'"
492+
)
493+
})?)?;
494+
let parquet_options = proto_sink
495+
.parquet_options
496+
.as_ref()
497+
.ok_or_else(|| {
498+
datafusion_common::internal_datafusion_err!(
499+
"ParquetSink is missing required field 'parquet_options'"
500+
)
501+
})?
502+
.try_into()?;
503+
let data_sink = ParquetSink::new(config, parquet_options);
504+
505+
let sink_schema = input.schema();
506+
let sort_order =
507+
parse_sink_sort_order(sink_node.sort_order.as_ref(), ctx, &sink_schema)?;
508+
509+
Ok(Arc::new(DataSinkExec::new(
510+
input,
511+
Arc::new(data_sink),
512+
sort_order,
513+
)))
514+
}
414515
}
415516

416517
/// Consumes a stream of [ArrowLeafColumn] via a channel and serializes them using an [ArrowColumnWriter]

‎datafusion/datasource/src/file_scan_config/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ pub(crate) mod sort_pushdown;
2424
/// Attaches inherent `to_proto_conf` / `from_proto_conf` / `parse_table_schema_from_proto`
2525
/// helpers to [`FileScanConfig`] used by every file source's `try_to_proto` hook.
2626
#[cfg(feature = "proto")]
27-
mod proto;
27+
pub(crate) mod proto;
2828

2929
use crate::file_groups::FileGroup;
3030
use crate::{

‎datafusion/datasource/src/file_scan_config/proto.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -458,7 +458,9 @@ fn file_group_from_proto(group: &protobuf::FileGroup) -> Result<FileGroup> {
458458
Ok(FileGroup::new(files))
459459
}
460460

461-
fn partitioned_file_to_proto(pf: &PartitionedFile) -> Result<protobuf::PartitionedFile> {
461+
pub(crate) fn partitioned_file_to_proto(
462+
pf: &PartitionedFile,
463+
) -> Result<protobuf::PartitionedFile> {
462464
let last_modified = pf.object_meta.last_modified;
463465
let last_modified_ns = last_modified.timestamp_nanos_opt().ok_or_else(|| {
464466
DataFusionError::Plan(format!(
@@ -487,7 +489,7 @@ fn partitioned_file_to_proto(pf: &PartitionedFile) -> Result<protobuf::Partition
487489
})
488490
}
489491

490-
fn partitioned_file_from_proto(
492+
pub(crate) fn partitioned_file_from_proto(
491493
val: &protobuf::PartitionedFile,
492494
) -> Result<PartitionedFile> {
493495
let mut pf = PartitionedFile::new_from_meta(ObjectMeta {

datafusion/datasource/src/file_sink_config.rs renamed to datafusion/datasource/src/file_sink_config/mod.rs

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,18 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
/// Shared `FileSinkConfig` <-> proto conversion, gated on the `proto` feature.
19+
/// Attaches inherent `to_proto` / `from_proto` helpers to [`FileSinkConfig`]
20+
/// (plus a `parse_sink_sort_order` helper) used by every file sink's
21+
/// `try_to_proto` / `try_from_proto` hook.
22+
#[cfg(feature = "proto")]
23+
pub(crate) mod proto;
24+
25+
/// Re-exported so each file sink's `try_from_proto` can decode the shared,
26+
/// optional required output ordering without naming the internal `proto` module.
27+
#[cfg(feature = "proto")]
28+
pub use proto::parse_sink_sort_order;
29+
1830
use std::sync::Arc;
1931

2032
use crate::ListingTableUrl;

0 commit comments

Comments
 (0)