Skip to content

Commit c1cb715

Browse files
authored
refactor(proto): migrate CsvSource serde (apache#24177)
## Which issue does this PR close? - Part of apache#23516. ## Rationale for this change Part of epic apache#23494. Moves `CsvSource` protobuf serialization from the central dispatch into the source implementation. ## What changes are included in this PR? Add protobuf serialization and deserialization to `CsvSource`, including the CSV format options represented by the existing wire format. Repoint the live decode arm to `CsvSource::try_from_proto` and remove the old central encode arm. Keep `try_into_csv_scan_physical_plan` as a deprecated compatibility wrapper that delegates to the new implementation. The protobuf wire format remains unchanged. ## Are these changes tested? Yes. Added `roundtrip_csv_scan_preserves_format_options`, covering header, delimiter, quote, escape, comment, multiline values, and truncated rows. ## Are there any user-facing changes? The existing `PhysicalPlanNodeExt` method remains available as a deprecated compatibility wrapper. There is no immediate API removal or wire-format change.
1 parent 038bfa2 commit c1cb715

5 files changed

Lines changed: 211 additions & 113 deletions

File tree

‎datafusion/datasource-csv/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ version.workspace = true
3131
all-features = true
3232

3333
[features]
34+
# Enables protobuf serialization hooks for CSV sources and sinks.
3435
proto = [
3536
"dep:datafusion-proto-models",
3637
"datafusion-datasource/proto",

‎datafusion/datasource-csv/src/source.rs‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,49 @@ impl FileSource for CsvSource {
308308
DisplayFormatType::TreeRender => Ok(()),
309309
}
310310
}
311+
312+
/// Emit a `CsvScan` node wrapping the shared base config and CSV options.
313+
#[cfg(feature = "proto")]
314+
fn try_to_proto(
315+
&self,
316+
base: &FileScanConfig,
317+
ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
318+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
319+
use datafusion_proto_models::protobuf;
320+
use protobuf::physical_plan_node::PhysicalPlanType;
321+
322+
let node = protobuf::CsvScanExecNode {
323+
base_conf: Some(base.try_to_proto(ctx)?),
324+
has_header: self.has_header(),
325+
delimiter: proto_byte_to_string(self.delimiter(), "delimiter")?,
326+
quote: proto_byte_to_string(self.quote(), "quote")?,
327+
optional_escape: self
328+
.escape()
329+
.map(|escape| {
330+
Ok::<_, DataFusionError>(
331+
protobuf::csv_scan_exec_node::OptionalEscape::Escape(
332+
proto_byte_to_string(escape, "escape")?,
333+
),
334+
)
335+
})
336+
.transpose()?,
337+
optional_comment: self
338+
.comment()
339+
.map(|comment| {
340+
Ok::<_, DataFusionError>(
341+
protobuf::csv_scan_exec_node::OptionalComment::Comment(
342+
proto_byte_to_string(comment, "comment")?,
343+
),
344+
)
345+
})
346+
.transpose()?,
347+
newlines_in_values: self.newlines_in_values(),
348+
truncate_rows: self.truncate_rows(),
349+
};
350+
Ok(Some(protobuf::PhysicalPlanNode {
351+
physical_plan_type: Some(PhysicalPlanType::CsvScan(node)),
352+
}))
353+
}
311354
}
312355

313356
impl FileOpener for CsvOpener {
@@ -501,3 +544,97 @@ pub async fn plan_to_csv(
501544

502545
Ok(())
503546
}
547+
548+
#[cfg(feature = "proto")]
549+
fn proto_byte_to_string(b: u8, description: &str) -> Result<String> {
550+
let bytes = &[b];
551+
let s = std::str::from_utf8(bytes).map_err(|_| {
552+
datafusion_common::internal_datafusion_err!(
553+
"Invalid CSV {description}: can not represent {bytes:0x?} as utf8"
554+
)
555+
})?;
556+
Ok(s.to_owned())
557+
}
558+
559+
#[cfg(feature = "proto")]
560+
fn proto_str_to_byte(s: &str, description: &str) -> Result<u8> {
561+
datafusion_common::assert_eq_or_internal_err!(
562+
s.len(),
563+
1,
564+
"Invalid CSV {description}: expected single character, got {s}"
565+
);
566+
Ok(s.as_bytes()[0])
567+
}
568+
569+
#[cfg(feature = "proto")]
570+
impl CsvSource {
571+
/// Reconstructs a `DataSourceExec` from a protobuf `CsvScan`.
572+
///
573+
/// Custom line terminators are not represented in the wire format.
574+
pub fn try_from_proto(
575+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
576+
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
577+
) -> Result<Arc<dyn ExecutionPlan>> {
578+
use datafusion_common::config::CsvOptions;
579+
use datafusion_datasource::file_compression_type::FileCompressionType;
580+
use datafusion_datasource::file_scan_config::{
581+
FileScanConfig, FileScanConfigBuilder,
582+
};
583+
use datafusion_datasource::source::DataSourceExec;
584+
use datafusion_proto_models::protobuf;
585+
586+
let scan = match &node.physical_plan_type {
587+
Some(protobuf::physical_plan_node::PhysicalPlanType::CsvScan(scan)) => scan,
588+
_ => {
589+
return datafusion_common::internal_err!(
590+
"PhysicalPlanNode is not a CsvScan"
591+
);
592+
}
593+
};
594+
595+
let base_conf = scan.base_conf.as_ref().ok_or_else(|| {
596+
datafusion_common::internal_datafusion_err!(
597+
"CsvScanExecNode is missing required field 'base_conf'"
598+
)
599+
})?;
600+
601+
let escape = match &scan.optional_escape {
602+
Some(protobuf::csv_scan_exec_node::OptionalEscape::Escape(escape)) => {
603+
Some(proto_str_to_byte(escape, "escape")?)
604+
}
605+
None => None,
606+
};
607+
let comment = match &scan.optional_comment {
608+
Some(protobuf::csv_scan_exec_node::OptionalComment::Comment(comment)) => {
609+
Some(proto_str_to_byte(comment, "comment")?)
610+
}
611+
None => None,
612+
};
613+
614+
let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?;
615+
616+
let csv_options = CsvOptions {
617+
has_header: Some(scan.has_header),
618+
delimiter: proto_str_to_byte(&scan.delimiter, "delimiter")?,
619+
quote: proto_str_to_byte(&scan.quote, "quote")?,
620+
newlines_in_values: Some(scan.newlines_in_values),
621+
truncated_rows: Some(scan.truncate_rows),
622+
..Default::default()
623+
};
624+
let source = Arc::new(
625+
CsvSource::new(table_schema)
626+
.with_csv_options(csv_options)
627+
.with_escape(escape)
628+
.with_comment(comment),
629+
);
630+
631+
// The compression type is not on the wire; CSV scans always
632+
// deserialize as uncompressed.
633+
let conf = FileScanConfigBuilder::from(FileScanConfig::try_from_proto(
634+
base_conf, ctx, source,
635+
)?)
636+
.with_file_compression_type(FileCompressionType::UNCOMPRESSED)
637+
.build();
638+
Ok(DataSourceExec::from_data_source(conf))
639+
}
640+
}

‎datafusion/proto/src/common.rs‎

Lines changed: 0 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -15,27 +15,6 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
use datafusion_common::{Result, assert_eq_or_internal_err, internal_datafusion_err};
19-
20-
pub(crate) fn str_to_byte(s: &String, description: &str) -> Result<u8> {
21-
assert_eq_or_internal_err!(
22-
s.len(),
23-
1,
24-
"Invalid CSV {description}: expected single character, got {s}"
25-
);
26-
Ok(s.as_bytes()[0])
27-
}
28-
29-
pub(crate) fn byte_to_string(b: u8, description: &str) -> Result<String> {
30-
let b = &[b];
31-
let b = std::str::from_utf8(b).map_err(|_| {
32-
internal_datafusion_err!(
33-
"Invalid CSV {description}: can not represent {b:0x?} as utf8"
34-
)
35-
})?;
36-
Ok(b.to_owned())
37-
}
38-
3918
#[macro_export]
4019
macro_rules! convert_required {
4120
($PB:expr) => {{

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 15 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,12 @@ use std::sync::Arc;
2323

2424
use arrow::datatypes::{IntervalMonthDayNanoType, Schema, SchemaRef};
2525
use datafusion_catalog::memory::MemorySourceConfig;
26-
use datafusion_common::config::CsvOptions;
2726
use datafusion_common::{
2827
DataFusionError, Result, internal_datafusion_err, internal_err, not_impl_err,
2928
};
3029
#[cfg(feature = "parquet")]
3130
use datafusion_datasource::file::FileSource;
32-
use datafusion_datasource::file_compression_type::FileCompressionType;
33-
use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};
31+
use datafusion_datasource::file_scan_config::FileScanConfig;
3432
use datafusion_datasource::sink::DataSinkExec;
3533
use datafusion_datasource::source::{DataSource, DataSourceExec};
3634
use datafusion_datasource_arrow::source::ArrowSource;
@@ -94,7 +92,6 @@ use datafusion_physical_plan::{ExecutionPlan, PhysicalExpr};
9492
use prost::Message;
9593
use prost::bytes::BufMut;
9694

97-
use crate::common::{byte_to_string, str_to_byte};
9895
use crate::convert_required;
9996
use crate::physical_plan::from_proto::{
10097
parse_physical_expr_with_converter, parse_physical_sort_exprs,
@@ -126,6 +123,7 @@ mod file_scan_config_serde {
126123
use datafusion_common::{Constraint, Constraints, ScalarValue, Statistics};
127124
use datafusion_datasource::file::FileSource;
128125
use datafusion_datasource::file_groups::FileGroup;
126+
use datafusion_datasource::file_scan_config::FileScanConfigBuilder;
129127
use datafusion_datasource::file_stream::FileOpener;
130128
use datafusion_datasource::{PartitionedFile, TableSchema};
131129
use datafusion_execution::object_store::ObjectStoreUrl;
@@ -1084,8 +1082,8 @@ pub trait PhysicalPlanNodeExt: Sized {
10841082
PhysicalPlanType::Filter(_) => {
10851083
FilterExec::try_from_proto(self.node(), &decode_ctx)
10861084
}
1087-
PhysicalPlanType::CsvScan(scan) => {
1088-
self.try_into_csv_scan_physical_plan(scan, ctx, proto_converter)
1085+
PhysicalPlanType::CsvScan(_) => {
1086+
CsvSource::try_from_proto(self.node(), &decode_ctx)
10891087
}
10901088
PhysicalPlanType::JsonScan(scan) => {
10911089
self.try_into_json_scan_physical_plan(scan, ctx, proto_converter)
@@ -1340,57 +1338,25 @@ pub trait PhysicalPlanNodeExt: Sized {
13401338
FilterExec::try_from_proto(&node, &decode_ctx)
13411339
}
13421340

1341+
#[deprecated(
1342+
since = "55.0.0",
1343+
note = "unused by DataFusion; `CsvSource` deserializes itself via `CsvSource::try_from_proto`"
1344+
)]
13431345
fn try_into_csv_scan_physical_plan(
13441346
&self,
13451347
scan: &protobuf::CsvScanExecNode,
13461348
ctx: &PhysicalPlanDecodeContext<'_>,
13471349
proto_converter: &dyn PhysicalProtoConverterExtension,
13481350
) -> Result<Arc<dyn ExecutionPlan>> {
1349-
let escape =
1350-
if let Some(protobuf::csv_scan_exec_node::OptionalEscape::Escape(escape)) =
1351-
&scan.optional_escape
1352-
{
1353-
Some(str_to_byte(escape, "escape")?)
1354-
} else {
1355-
None
1356-
};
1357-
1358-
let comment = if let Some(
1359-
protobuf::csv_scan_exec_node::OptionalComment::Comment(comment),
1360-
) = &scan.optional_comment
1361-
{
1362-
Some(str_to_byte(comment, "comment")?)
1363-
} else {
1364-
None
1365-
};
1366-
1367-
// Parse table schema with partition columns
1368-
let table_schema =
1369-
parse_table_schema_from_proto(scan.base_conf.as_ref().unwrap())?;
1370-
1371-
let csv_options = CsvOptions {
1372-
has_header: Some(scan.has_header),
1373-
delimiter: str_to_byte(&scan.delimiter, "delimiter")?,
1374-
quote: str_to_byte(&scan.quote, "quote")?,
1375-
newlines_in_values: Some(scan.newlines_in_values),
1376-
..Default::default()
1377-
};
1378-
let source = Arc::new(
1379-
CsvSource::new(table_schema)
1380-
.with_csv_options(csv_options)
1381-
.with_escape(escape)
1382-
.with_comment(comment),
1383-
);
1384-
1385-
let conf = FileScanConfigBuilder::from(parse_protobuf_file_scan_config(
1386-
scan.base_conf.as_ref().unwrap(),
1351+
let node = protobuf::PhysicalPlanNode {
1352+
physical_plan_type: Some(PhysicalPlanType::CsvScan(scan.clone())),
1353+
};
1354+
let decoder = ConverterPlanDecoder {
13871355
ctx,
13881356
proto_converter,
1389-
source,
1390-
)?)
1391-
.with_file_compression_type(FileCompressionType::UNCOMPRESSED)
1392-
.build();
1393-
Ok(DataSourceExec::from_data_source(conf))
1357+
};
1358+
let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
1359+
CsvSource::try_from_proto(&node, &decode_ctx)
13941360
}
13951361

13961362
fn try_into_json_scan_physical_plan(
@@ -2561,47 +2527,6 @@ pub trait PhysicalPlanNodeExt: Sized {
25612527
proto_converter: &dyn PhysicalProtoConverterExtension,
25622528
) -> Result<Option<protobuf::PhysicalPlanNode>> {
25632529
let data_source = data_source_exec.data_source();
2564-
if let Some(maybe_csv) = data_source.downcast_ref::<FileScanConfig>() {
2565-
let source = maybe_csv.file_source();
2566-
if let Some(csv_config) = source.downcast_ref::<CsvSource>() {
2567-
return Ok(Some(protobuf::PhysicalPlanNode {
2568-
physical_plan_type: Some(PhysicalPlanType::CsvScan(
2569-
protobuf::CsvScanExecNode {
2570-
base_conf: Some(serialize_file_scan_config(
2571-
maybe_csv,
2572-
codec,
2573-
proto_converter,
2574-
)?),
2575-
has_header: csv_config.has_header(),
2576-
delimiter: byte_to_string(
2577-
csv_config.delimiter(),
2578-
"delimiter",
2579-
)?,
2580-
quote: byte_to_string(csv_config.quote(), "quote")?,
2581-
optional_escape: if let Some(escape) = csv_config.escape() {
2582-
Some(
2583-
protobuf::csv_scan_exec_node::OptionalEscape::Escape(
2584-
byte_to_string(escape, "escape")?,
2585-
),
2586-
)
2587-
} else {
2588-
None
2589-
},
2590-
optional_comment: if let Some(comment) = csv_config.comment()
2591-
{
2592-
Some(protobuf::csv_scan_exec_node::OptionalComment::Comment(
2593-
byte_to_string(comment, "comment")?,
2594-
))
2595-
} else {
2596-
None
2597-
},
2598-
newlines_in_values: csv_config.newlines_in_values(),
2599-
truncate_rows: csv_config.truncate_rows(),
2600-
},
2601-
)),
2602-
}));
2603-
}
2604-
}
26052530

26062531
if let Some(scan_conf) = data_source.downcast_ref::<FileScanConfig>() {
26072532
let source = scan_conf.file_source();

0 commit comments

Comments
 (0)