Skip to content

Commit fe80479

Browse files
committed
fix(proto): restore sink conversions
Moving protobuf models out of datafusion-proto made its standard TryFrom implementations violate Rust's orphan rules. Replacing them with TryFromProto silently removed APIs shipped in 54.1.0. Define standard conversions in each sink's owning crate and keep the legacy traits as delegates so one mapping serves both call paths. Refs apache#23519 Refs apache#24019 Signed-off-by: Jiawei Zhao <[email protected]>
1 parent 4998d4b commit fe80479

6 files changed

Lines changed: 141 additions & 119 deletions

File tree

‎datafusion/datasource-csv/src/file_format.rs‎

Lines changed: 44 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -837,10 +837,7 @@ impl DataSink for CsvSink {
837837

838838
let input = ctx.encode_child(exec.input())?;
839839
let sort_order = exec.encode_sort_order(ctx)?;
840-
let sink = protobuf::CsvSink {
841-
config: Some((&self.config).try_into()?),
842-
writer_options: Some(self.writer_options().try_into()?),
843-
};
840+
let sink = protobuf::CsvSink::try_from(self)?;
844841
let node = protobuf::CsvSinkExecNode {
845842
input: Some(Box::new(input)),
846843
sink: Some(sink),
@@ -853,6 +850,43 @@ impl DataSink for CsvSink {
853850
}
854851
}
855852

853+
#[cfg(feature = "proto")]
854+
impl TryFrom<&CsvSink> for datafusion_proto_models::protobuf::CsvSink {
855+
type Error = DataFusionError;
856+
857+
fn try_from(value: &CsvSink) -> Result<Self> {
858+
Ok(Self {
859+
config: Some(value.config().try_into()?),
860+
writer_options: Some(value.writer_options().try_into()?),
861+
})
862+
}
863+
}
864+
865+
#[cfg(feature = "proto")]
866+
impl TryFrom<&datafusion_proto_models::protobuf::CsvSink> for CsvSink {
867+
type Error = DataFusionError;
868+
869+
fn try_from(value: &datafusion_proto_models::protobuf::CsvSink) -> Result<Self> {
870+
let config =
871+
FileSinkConfig::try_from(value.config.as_ref().ok_or_else(|| {
872+
datafusion_common::internal_datafusion_err!(
873+
"CsvSink is missing required field 'config'"
874+
)
875+
})?)?;
876+
let writer_options = value
877+
.writer_options
878+
.as_ref()
879+
.ok_or_else(|| {
880+
datafusion_common::internal_datafusion_err!(
881+
"CsvSink is missing required field 'writer_options'"
882+
)
883+
})?
884+
.try_into()?;
885+
886+
Ok(Self::new(config, writer_options))
887+
}
888+
}
889+
856890
#[cfg(feature = "proto")]
857891
impl CsvSink {
858892
/// Reconstructs a [`DataSinkExec`] containing a `CsvSink` from protobuf.
@@ -862,16 +896,11 @@ impl CsvSink {
862896
) -> Result<Arc<dyn ExecutionPlan>> {
863897
use datafusion_proto_models::protobuf;
864898

865-
let sink_node = match &node.physical_plan_type {
866-
Some(protobuf::physical_plan_node::PhysicalPlanType::CsvSink(sink)) => {
867-
sink.as_ref()
868-
}
869-
_ => {
870-
return datafusion_common::internal_err!(
871-
"PhysicalPlanNode is not a CsvSink"
872-
);
873-
}
874-
};
899+
let sink_node = datafusion_physical_plan::expect_plan_variant!(
900+
node,
901+
protobuf::physical_plan_node::PhysicalPlanType::CsvSink,
902+
"CsvSink",
903+
);
875904
let input = ctx.decode_required_child(
876905
sink_node.input.as_deref(),
877906
"CsvSinkExecNode",
@@ -882,22 +911,7 @@ impl CsvSink {
882911
"CsvSinkExecNode is missing required field 'sink'"
883912
)
884913
})?;
885-
let config =
886-
FileSinkConfig::try_from(proto_sink.config.as_ref().ok_or_else(|| {
887-
datafusion_common::internal_datafusion_err!(
888-
"CsvSink is missing required field 'config'"
889-
)
890-
})?)?;
891-
let writer_options = proto_sink
892-
.writer_options
893-
.as_ref()
894-
.ok_or_else(|| {
895-
datafusion_common::internal_datafusion_err!(
896-
"CsvSink is missing required field 'writer_options'"
897-
)
898-
})?
899-
.try_into()?;
900-
let data_sink = CsvSink::new(config, writer_options);
914+
let data_sink = CsvSink::try_from(proto_sink)?;
901915
let sort_order = DataSinkExec::decode_sort_order(
902916
sink_node.sort_order.as_ref(),
903917
ctx,

‎datafusion/datasource-json/src/file_format.rs‎

Lines changed: 44 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -502,10 +502,7 @@ impl DataSink for JsonSink {
502502

503503
let input = ctx.encode_child(exec.input())?;
504504
let sort_order = exec.encode_sort_order(ctx)?;
505-
let sink = protobuf::JsonSink {
506-
config: Some((&self.config).try_into()?),
507-
writer_options: Some(self.writer_options().try_into()?),
508-
};
505+
let sink = protobuf::JsonSink::try_from(self)?;
509506
let node = protobuf::JsonSinkExecNode {
510507
input: Some(Box::new(input)),
511508
sink: Some(sink),
@@ -518,6 +515,43 @@ impl DataSink for JsonSink {
518515
}
519516
}
520517

518+
#[cfg(feature = "proto")]
519+
impl TryFrom<&JsonSink> for datafusion_proto_models::protobuf::JsonSink {
520+
type Error = datafusion_common::DataFusionError;
521+
522+
fn try_from(value: &JsonSink) -> Result<Self> {
523+
Ok(Self {
524+
config: Some(value.config().try_into()?),
525+
writer_options: Some(value.writer_options().try_into()?),
526+
})
527+
}
528+
}
529+
530+
#[cfg(feature = "proto")]
531+
impl TryFrom<&datafusion_proto_models::protobuf::JsonSink> for JsonSink {
532+
type Error = datafusion_common::DataFusionError;
533+
534+
fn try_from(value: &datafusion_proto_models::protobuf::JsonSink) -> Result<Self> {
535+
let config =
536+
FileSinkConfig::try_from(value.config.as_ref().ok_or_else(|| {
537+
datafusion_common::internal_datafusion_err!(
538+
"JsonSink is missing required field 'config'"
539+
)
540+
})?)?;
541+
let writer_options = value
542+
.writer_options
543+
.as_ref()
544+
.ok_or_else(|| {
545+
datafusion_common::internal_datafusion_err!(
546+
"JsonSink is missing required field 'writer_options'"
547+
)
548+
})?
549+
.try_into()?;
550+
551+
Ok(Self::new(config, writer_options))
552+
}
553+
}
554+
521555
#[cfg(feature = "proto")]
522556
impl JsonSink {
523557
/// Reconstructs a [`DataSinkExec`] containing a `JsonSink` from protobuf.
@@ -527,16 +561,11 @@ impl JsonSink {
527561
) -> Result<Arc<dyn ExecutionPlan>> {
528562
use datafusion_proto_models::protobuf;
529563

530-
let sink_node = match &node.physical_plan_type {
531-
Some(protobuf::physical_plan_node::PhysicalPlanType::JsonSink(sink)) => {
532-
sink.as_ref()
533-
}
534-
_ => {
535-
return datafusion_common::internal_err!(
536-
"PhysicalPlanNode is not a JsonSink"
537-
);
538-
}
539-
};
564+
let sink_node = datafusion_physical_plan::expect_plan_variant!(
565+
node,
566+
protobuf::physical_plan_node::PhysicalPlanType::JsonSink,
567+
"JsonSink",
568+
);
540569
let input = ctx.decode_required_child(
541570
sink_node.input.as_deref(),
542571
"JsonSinkExecNode",
@@ -547,22 +576,7 @@ impl JsonSink {
547576
"JsonSinkExecNode is missing required field 'sink'"
548577
)
549578
})?;
550-
let config =
551-
FileSinkConfig::try_from(proto_sink.config.as_ref().ok_or_else(|| {
552-
datafusion_common::internal_datafusion_err!(
553-
"JsonSink is missing required field 'config'"
554-
)
555-
})?)?;
556-
let writer_options = proto_sink
557-
.writer_options
558-
.as_ref()
559-
.ok_or_else(|| {
560-
datafusion_common::internal_datafusion_err!(
561-
"JsonSink is missing required field 'writer_options'"
562-
)
563-
})?
564-
.try_into()?;
565-
let data_sink = JsonSink::new(config, writer_options);
579+
let data_sink = JsonSink::try_from(proto_sink)?;
566580
let sort_order = DataSinkExec::decode_sort_order(
567581
sink_node.sort_order.as_ref(),
568582
ctx,

‎datafusion/datasource-parquet/src/sink.rs‎

Lines changed: 44 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -425,10 +425,7 @@ impl DataSink for ParquetSink {
425425

426426
let input = ctx.encode_child(exec.input())?;
427427
let sort_order = exec.encode_sort_order(ctx)?;
428-
let sink = protobuf::ParquetSink {
429-
config: Some((&self.config).try_into()?),
430-
parquet_options: Some(self.parquet_options().try_into()?),
431-
};
428+
let sink = protobuf::ParquetSink::try_from(self)?;
432429
let node = protobuf::ParquetSinkExecNode {
433430
input: Some(Box::new(input)),
434431
sink: Some(sink),
@@ -441,6 +438,43 @@ impl DataSink for ParquetSink {
441438
}
442439
}
443440

441+
#[cfg(feature = "proto")]
442+
impl TryFrom<&ParquetSink> for datafusion_proto_models::protobuf::ParquetSink {
443+
type Error = DataFusionError;
444+
445+
fn try_from(value: &ParquetSink) -> Result<Self> {
446+
Ok(Self {
447+
config: Some(value.config().try_into()?),
448+
parquet_options: Some(value.parquet_options().try_into()?),
449+
})
450+
}
451+
}
452+
453+
#[cfg(feature = "proto")]
454+
impl TryFrom<&datafusion_proto_models::protobuf::ParquetSink> for ParquetSink {
455+
type Error = DataFusionError;
456+
457+
fn try_from(value: &datafusion_proto_models::protobuf::ParquetSink) -> Result<Self> {
458+
let config =
459+
FileSinkConfig::try_from(value.config.as_ref().ok_or_else(|| {
460+
datafusion_common::internal_datafusion_err!(
461+
"ParquetSink is missing required field 'config'"
462+
)
463+
})?)?;
464+
let parquet_options = value
465+
.parquet_options
466+
.as_ref()
467+
.ok_or_else(|| {
468+
datafusion_common::internal_datafusion_err!(
469+
"ParquetSink is missing required field 'parquet_options'"
470+
)
471+
})?
472+
.try_into()?;
473+
474+
Ok(Self::new(config, parquet_options))
475+
}
476+
}
477+
444478
#[cfg(feature = "proto")]
445479
impl ParquetSink {
446480
/// Reconstructs a [`DataSinkExec`] containing a `ParquetSink` from protobuf.
@@ -450,16 +484,11 @@ impl ParquetSink {
450484
) -> Result<Arc<dyn ExecutionPlan>> {
451485
use datafusion_proto_models::protobuf;
452486

453-
let sink_node = match &node.physical_plan_type {
454-
Some(protobuf::physical_plan_node::PhysicalPlanType::ParquetSink(sink)) => {
455-
sink.as_ref()
456-
}
457-
_ => {
458-
return datafusion_common::internal_err!(
459-
"PhysicalPlanNode is not a ParquetSink"
460-
);
461-
}
462-
};
487+
let sink_node = datafusion_physical_plan::expect_plan_variant!(
488+
node,
489+
protobuf::physical_plan_node::PhysicalPlanType::ParquetSink,
490+
"ParquetSink",
491+
);
463492
let input = ctx.decode_required_child(
464493
sink_node.input.as_deref(),
465494
"ParquetSinkExecNode",
@@ -470,22 +499,7 @@ impl ParquetSink {
470499
"ParquetSinkExecNode is missing required field 'sink'"
471500
)
472501
})?;
473-
let config =
474-
FileSinkConfig::try_from(proto_sink.config.as_ref().ok_or_else(|| {
475-
datafusion_common::internal_datafusion_err!(
476-
"ParquetSink is missing required field 'config'"
477-
)
478-
})?)?;
479-
let parquet_options = proto_sink
480-
.parquet_options
481-
.as_ref()
482-
.ok_or_else(|| {
483-
datafusion_common::internal_datafusion_err!(
484-
"ParquetSink is missing required field 'parquet_options'"
485-
)
486-
})?
487-
.try_into()?;
488-
let data_sink = ParquetSink::new(config, parquet_options);
502+
let data_sink = ParquetSink::try_from(proto_sink)?;
489503
let sort_order = DataSinkExec::decode_sort_order(
490504
sink_node.sort_order.as_ref(),
491505
ctx,

‎datafusion/proto/src/physical_plan/from_proto.rs‎

Lines changed: 4 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ use super::{
5757
};
5858
use crate::convert::TryFromProto;
5959
use crate::protobuf::physical_expr_node::ExprType;
60-
use crate::{convert_required, convert_required_proto, protobuf};
60+
use crate::{convert_required, protobuf};
6161
use datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr;
6262

6363
/// Parses a physical sort expression from a protobuf.
@@ -586,10 +586,7 @@ impl TryFromProto<&protobuf::JsonSink> for JsonSink {
586586
type Error = DataFusionError;
587587

588588
fn try_from_proto(value: &protobuf::JsonSink) -> Result<Self, Self::Error> {
589-
Ok(Self::new(
590-
convert_required_proto!(FileSinkConfig, value.config)?,
591-
convert_required!(value.writer_options)?,
592-
))
589+
Self::try_from(value)
593590
}
594591
}
595592

@@ -598,21 +595,15 @@ impl TryFromProto<&protobuf::ParquetSink> for ParquetSink {
598595
type Error = DataFusionError;
599596

600597
fn try_from_proto(value: &protobuf::ParquetSink) -> Result<Self, Self::Error> {
601-
Ok(Self::new(
602-
convert_required_proto!(FileSinkConfig, value.config)?,
603-
convert_required!(value.parquet_options)?,
604-
))
598+
Self::try_from(value)
605599
}
606600
}
607601

608602
impl TryFromProto<&protobuf::CsvSink> for CsvSink {
609603
type Error = DataFusionError;
610604

611605
fn try_from_proto(value: &protobuf::CsvSink) -> Result<Self, Self::Error> {
612-
Ok(Self::new(
613-
convert_required_proto!(FileSinkConfig, value.config)?,
614-
convert_required!(value.writer_options)?,
615-
))
606+
Self::try_from(value)
616607
}
617608
}
618609

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -793,9 +793,7 @@ pub trait PhysicalPlanNodeExt: Sized {
793793
ParquetSink::try_from_proto(self.node(), &decode_ctx)
794794
}
795795
#[cfg(not(feature = "parquet"))]
796-
panic!(
797-
"Unable to process a Parquet PhysicalPlan when `parquet` feature is not enabled"
798-
)
796+
panic!("Trying to use ParquetSink without `parquet` feature enabled")
799797
}
800798
PhysicalPlanType::Unnest(_) => {
801799
UnnestExec::try_from_proto(self.node(), &decode_ctx)

0 commit comments

Comments
 (0)