Skip to content

Commit 97e1c1c

Browse files
authored
fix(proto): preserve CSV/JSON scan options on roundtrip (apache#24233)
## Which issue does this PR close? - Closes apache#24180. ## Rationale for this change Physical-plan protobuf serialization does not preserve several CSV and JSON scan options. Custom CSV terminators, JSON newline-delimited mode, and file compression therefore revert to their defaults after a roundtrip, which can cause the decoded plan to read the file incorrectly. ## What changes are included in this PR? - Add `terminator` to `CsvScanExecNode`. - Add `newline_delimited` to `JsonScanExecNode`. - Add `file_compression_type` to the shared `FileScanExecConf`. - Serialize and restore these options in CSV and JSON scans. ## Are these changes tested? Yes. Extended the CSV and JSON physical-plan roundtrip tests to cover custom terminators, non-newline-delimited JSON, compression, and backward-compatible defaults. ## Are there any user-facing changes? CSV and JSON scans now preserve their format and compression options across protobuf roundtrips. The protobuf changes are additive and backward compatible. `JsonSource` also gains an `is_newline_delimited` getter.
1 parent f10331f commit 97e1c1c

10 files changed

Lines changed: 317 additions & 28 deletions

File tree

‎Cargo.lock‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/datasource-csv/src/source.rs‎

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,7 @@ impl FileSource for CsvSource {
356356
.transpose()?,
357357
newlines_in_values: self.newlines_in_values(),
358358
truncate_rows: self.truncate_rows(),
359+
terminator: self.terminator().map(|terminator| vec![terminator]),
359360
};
360361
Ok(Some(protobuf::PhysicalPlanNode {
361362
physical_plan_type: Some(PhysicalPlanType::CsvScan(node)),
@@ -576,20 +577,28 @@ fn proto_str_to_byte(s: &str, description: &str) -> Result<u8> {
576577
Ok(s.as_bytes()[0])
577578
}
578579

580+
#[cfg(feature = "proto")]
581+
fn proto_bytes_to_byte(bytes: &[u8], description: &str) -> Result<u8> {
582+
let [byte] = bytes else {
583+
return datafusion_common::internal_err!(
584+
"Invalid CSV {description}: expected exactly one byte, got {}",
585+
bytes.len()
586+
);
587+
};
588+
Ok(*byte)
589+
}
590+
579591
#[cfg(feature = "proto")]
580592
impl CsvSource {
581593
/// Reconstructs a `DataSourceExec` from a protobuf `CsvScan`.
582594
///
583-
/// Custom line terminators are not represented in the wire format.
595+
/// Payloads without a terminator use the default newline terminator.
584596
pub fn try_from_proto(
585597
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
586598
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
587599
) -> Result<Arc<dyn ExecutionPlan>> {
588600
use datafusion_common::config::CsvOptions;
589-
use datafusion_datasource::file_compression_type::FileCompressionType;
590-
use datafusion_datasource::file_scan_config::{
591-
FileScanConfig, FileScanConfigBuilder,
592-
};
601+
use datafusion_datasource::file_scan_config::FileScanConfig;
593602
use datafusion_datasource::source::DataSourceExec;
594603
use datafusion_proto_models::protobuf;
595604

@@ -617,6 +626,11 @@ impl CsvSource {
617626
}
618627
None => None,
619628
};
629+
let terminator = scan
630+
.terminator
631+
.as_deref()
632+
.map(|terminator| proto_bytes_to_byte(terminator, "terminator"))
633+
.transpose()?;
620634

621635
let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?;
622636

@@ -632,16 +646,11 @@ impl CsvSource {
632646
CsvSource::new(table_schema)
633647
.with_csv_options(csv_options)
634648
.with_escape(escape)
635-
.with_comment(comment),
649+
.with_comment(comment)
650+
.with_terminator(terminator),
636651
);
637652

638-
// The compression type is not on the wire; CSV scans always
639-
// deserialize as uncompressed.
640-
let conf = FileScanConfigBuilder::from(FileScanConfig::try_from_proto(
641-
base_conf, ctx, source,
642-
)?)
643-
.with_file_compression_type(FileCompressionType::UNCOMPRESSED)
644-
.build();
653+
let conf = FileScanConfig::try_from_proto(base_conf, ctx, source)?;
645654
Ok(DataSourceExec::from_data_source(conf))
646655
}
647656
}

‎datafusion/datasource-json/src/source.rs‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,11 @@ impl JsonSource {
159159
self.newline_delimited = newline_delimited;
160160
self
161161
}
162+
163+
/// Returns whether this source reads newline-delimited JSON.
164+
pub fn is_newline_delimited(&self) -> bool {
165+
self.newline_delimited
166+
}
162167
}
163168

164169
impl From<JsonSource> for Arc<dyn FileSource> {
@@ -254,6 +259,11 @@ impl FileSource for JsonSource {
254259

255260
let node = protobuf::JsonScanExecNode {
256261
base_conf: Some(base.try_to_proto(ctx)?),
262+
newline_delimited: if self.newline_delimited {
263+
None
264+
} else {
265+
Some(false)
266+
},
257267
};
258268
Ok(Some(protobuf::PhysicalPlanNode {
259269
physical_plan_type: Some(PhysicalPlanType::JsonScan(node)),
@@ -265,7 +275,7 @@ impl FileSource for JsonSource {
265275
impl JsonSource {
266276
/// Reconstructs a `DataSourceExec` from a protobuf `JsonScan`.
267277
///
268-
/// Defaults to newline-delimited JSON because protobuf does not encode the mode.
278+
/// Payloads without a mode default to newline-delimited JSON.
269279
pub fn try_from_proto(
270280
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
271281
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
@@ -289,7 +299,10 @@ impl JsonSource {
289299
})?;
290300

291301
let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?;
292-
let source = Arc::new(JsonSource::new(table_schema));
302+
let source = Arc::new(
303+
JsonSource::new(table_schema)
304+
.with_newline_delimited(scan.newline_delimited.unwrap_or(true)),
305+
);
293306

294307
let conf = FileScanConfig::try_from_proto(base_conf, ctx, source)?;
295308
Ok(DataSourceExec::from_data_source(conf))

‎datafusion/datasource/src/file_scan_config/proto.rs‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@
2727
//! `FileSource::try_to_proto` hook (CSV, JSON, Arrow, Parquet, Avro) builds its
2828
//! `*ScanExecNode` around [`FileScanConfig::try_to_proto`] and decodes with
2929
//! [`FileScanConfig::try_from_proto`], keeping a single copy of the shared
30-
//! wire logic. The wire format is byte-for-byte identical to the old central
31-
//! serializer.
30+
//! wire logic. Existing fields remain wire-compatible with the old central
31+
//! serializer; new options use optional fields with legacy defaults.
3232
//!
3333
//! Child physical expressions (sort orderings, hash/range partitioning, and
3434
//! projection expressions) are (de)serialized through `ctx.encode_expr` /
@@ -38,6 +38,7 @@
3838
use std::sync::Arc;
3939

4040
use arrow::datatypes::Schema;
41+
use datafusion_common::parsers::CompressionTypeVariant;
4142
use datafusion_common::{DataFusionError, Result, internal_datafusion_err};
4243
use datafusion_execution::object_store::ObjectStoreUrl;
4344
use datafusion_physical_expr::projection::{ProjectionExpr, ProjectionExprs};
@@ -46,9 +47,11 @@ use datafusion_physical_expr_common::sort_expr::{
4647
sort_exprs_try_from_proto, sort_exprs_try_to_proto,
4748
};
4849
use datafusion_physical_plan::proto::{ExecutionPlanDecodeCtx, ExecutionPlanEncodeCtx};
50+
use datafusion_proto_models::datafusion_common::CompressionTypeVariant as ProtoCompressionTypeVariant;
4951
use datafusion_proto_models::protobuf;
5052

5153
use crate::file::FileSource;
54+
use crate::file_compression_type::FileCompressionType;
5255
use crate::file_scan_config::{FileScanConfig, FileScanConfigBuilder};
5356
use crate::table_schema::TableSchema;
5457

@@ -57,8 +60,9 @@ impl FileScanConfig {
5760
/// [`protobuf::FileScanExecConf`].
5861
///
5962
/// Each concrete [`FileSource::try_to_proto`]
60-
/// wraps the returned value in its own `*ScanExecNode`. Byte-compatible with
61-
/// the former `serialize_file_scan_config` in `datafusion-proto`.
63+
/// wraps the returned value in its own `*ScanExecNode`. Existing fields are
64+
/// byte-compatible with the former `serialize_file_scan_config` in
65+
/// `datafusion-proto`.
6266
pub fn try_to_proto(
6367
&self,
6468
ctx: &ExecutionPlanEncodeCtx<'_>,
@@ -114,6 +118,13 @@ impl FileScanConfig {
114118
})
115119
.transpose()?;
116120

121+
let file_compression_type =
122+
self.file_compression_type.is_compressed().then(|| {
123+
let compression: ProtoCompressionTypeVariant =
124+
(*self.file_compression_type.get_variant()).into();
125+
compression as i32
126+
});
127+
117128
Ok(protobuf::FileScanExecConf {
118129
file_groups,
119130
statistics: Some((&self.statistics()).into()),
@@ -131,14 +142,16 @@ impl FileScanConfig {
131142
batch_size: self.batch_size.map(|s| s as u64),
132143
projection_exprs,
133144
output_partitioning,
145+
file_compression_type,
134146
})
135147
}
136148

137149
/// Reconstruct a [`FileScanConfig`] from a [`protobuf::FileScanExecConf`]
138150
/// and a `file_source` the caller has already rebuilt (typically from the
139151
/// table schema via [`FileScanConfig::parse_table_schema_from_proto`]).
140152
///
141-
/// Byte-compatible with the former `parse_protobuf_file_scan_config`.
153+
/// Existing fields are byte-compatible with the former
154+
/// `parse_protobuf_file_scan_config`.
142155
pub fn try_from_proto(
143156
conf: &protobuf::FileScanExecConf,
144157
ctx: &ExecutionPlanDecodeCtx<'_>,
@@ -194,6 +207,19 @@ impl FileScanConfig {
194207
.transpose()?
195208
.flatten();
196209

210+
let file_compression_type = conf
211+
.file_compression_type
212+
.map(|value| {
213+
let compression =
214+
ProtoCompressionTypeVariant::try_from(value).map_err(|_| {
215+
internal_datafusion_err!("Unknown file compression type: {value}")
216+
})?;
217+
let compression: CompressionTypeVariant = compression.into();
218+
Ok::<_, DataFusionError>(FileCompressionType::from(compression))
219+
})
220+
.transpose()?
221+
.unwrap_or(FileCompressionType::UNCOMPRESSED);
222+
197223
// Parse projection expressions if present and apply to the file source.
198224
let file_source = if let Some(proto_projection_exprs) = &conf.projection_exprs {
199225
let projection_exprs: Vec<ProjectionExpr> = proto_projection_exprs
@@ -226,7 +252,8 @@ impl FileScanConfig {
226252
.with_limit(conf.limit.as_ref().map(|sl| sl.limit as usize))
227253
.with_output_ordering(output_ordering)
228254
.with_output_partitioning(output_partitioning)
229-
.with_batch_size(conf.batch_size.map(|s| s as usize));
255+
.with_batch_size(conf.batch_size.map(|s| s as usize))
256+
.with_file_compression_type(file_compression_type);
230257
Ok(config_builder.build())
231258
}
232259

‎datafusion/proto-models/proto/datafusion.proto‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1265,6 +1265,9 @@ message FileScanExecConf {
12651265
reserved 14;
12661266
reserved "partitioned_by_file_group";
12671267
optional Partitioning output_partitioning = 15;
1268+
// Compression used by formats such as CSV and JSON. Absent means uncompressed
1269+
// for compatibility with payloads written before this field existed.
1270+
optional datafusion_common.CompressionTypeVariant file_compression_type = 16;
12681271
}
12691272

12701273
message ParquetScanExecNode {
@@ -1291,10 +1294,14 @@ message CsvScanExecNode {
12911294
}
12921295
bool newlines_in_values = 7;
12931296
bool truncate_rows = 8;
1297+
// Custom one-byte line terminator. Absent means the default newline terminator.
1298+
optional bytes terminator = 9;
12941299
}
12951300

12961301
message JsonScanExecNode {
12971302
FileScanExecConf base_conf = 1;
1303+
// Absent means newline-delimited JSON for compatibility with older payloads.
1304+
optional bool newline_delimited = 2;
12981305
}
12991306

13001307
message AvroScanExecNode {

0 commit comments

Comments
 (0)