Skip to content

Commit 18ef6cf

Browse files
adriangbclaude
andcommitted
prototype: DataSource/FileSource proto hook + CsvSource migration (apache#22419 stack C)
Adds a proto feature + try_to_proto hook to the DataSource and FileSource traits (datafusion-datasource), so DataSourceExec rides the ExecutionPlan hook and delegates polymorphically: DataSourceExec -> DataSource -> FileSource. No downcast in the encode path for migrated sources. Ports the shared FileScanConfig <-> proto conversion into datafusion-datasource (FileScanConfig::{to_proto_conf, from_proto_conf, parse_table_schema_from_proto}); orderings/partition exprs go through the ctx, schema/stats/scalars via proto-common. Needs NO PhysicalExtensionCodec. CsvSource migrated as proof: its central encode branch is deleted and the CsvScan decode arm repointed to CsvSource::try_from_proto. New roundtrip_csv_scan test proves the hook chain end-to-end. All 191 proto_integration tests pass. Foundation confirms all 4 remaining formats (json/arrow/avro/parquet) are clean (parquet encryption is a non-issue: crypto isn't in the proto today). Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01SfBPC2yYCwHUn4V8pQCVj3
1 parent a6326e9 commit 18ef6cf

12 files changed

Lines changed: 822 additions & 48 deletions

File tree

‎Cargo.lock‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/datasource-csv/Cargo.toml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,15 @@ version.workspace = true
3030
[package.metadata.docs.rs]
3131
all-features = true
3232

33+
[features]
34+
# Enables `FileSource::try_to_proto` on `CsvSource` and the `CsvScan` decode
35+
# entry point. Mirrors the `proto` feature on `datafusion-datasource`.
36+
proto = [
37+
"dep:datafusion-proto-models",
38+
"datafusion-datasource/proto",
39+
"datafusion-physical-plan/proto",
40+
]
41+
3342
[dependencies]
3443
arrow = { workspace = true }
3544
async-trait = { workspace = true }
@@ -41,6 +50,7 @@ datafusion-execution = { workspace = true }
4150
datafusion-expr = { workspace = true }
4251
datafusion-physical-expr-common = { workspace = true }
4352
datafusion-physical-plan = { workspace = true }
53+
datafusion-proto-models = { workspace = true, optional = true }
4454
datafusion-session = { workspace = true }
4555
futures = { workspace = true }
4656
object_store = { workspace = true }

‎datafusion/datasource-csv/src/source.rs‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,51 @@ impl FileSource for CsvSource {
308308
DisplayFormatType::TreeRender => Ok(()),
309309
}
310310
}
311+
312+
/// Emit a `CsvScan` node wrapping the shared base config. Kept byte-identical
313+
/// to the former `try_from_data_source_exec` CsvScan branch in
314+
/// `datafusion-proto`.
315+
#[cfg(feature = "proto")]
316+
fn try_to_proto(
317+
&self,
318+
base: &FileScanConfig,
319+
ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
320+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
321+
use datafusion_proto_models::protobuf;
322+
use protobuf::physical_plan_node::PhysicalPlanType;
323+
324+
let node = protobuf::CsvScanExecNode {
325+
base_conf: Some(base.to_proto_conf(ctx)?),
326+
has_header: self.has_header(),
327+
delimiter: proto_byte_to_string(self.delimiter(), "delimiter")?,
328+
quote: proto_byte_to_string(self.quote(), "quote")?,
329+
optional_escape: self
330+
.escape()
331+
.map(|escape| {
332+
Ok::<_, DataFusionError>(
333+
protobuf::csv_scan_exec_node::OptionalEscape::Escape(
334+
proto_byte_to_string(escape, "escape")?,
335+
),
336+
)
337+
})
338+
.transpose()?,
339+
optional_comment: self
340+
.comment()
341+
.map(|comment| {
342+
Ok::<_, DataFusionError>(
343+
protobuf::csv_scan_exec_node::OptionalComment::Comment(
344+
proto_byte_to_string(comment, "comment")?,
345+
),
346+
)
347+
})
348+
.transpose()?,
349+
newlines_in_values: self.newlines_in_values(),
350+
truncate_rows: self.truncate_rows(),
351+
};
352+
Ok(Some(protobuf::PhysicalPlanNode {
353+
physical_plan_type: Some(PhysicalPlanType::CsvScan(node)),
354+
}))
355+
}
311356
}
312357

313358
impl FileOpener for CsvOpener {
@@ -501,3 +546,104 @@ pub async fn plan_to_csv(
501546

502547
Ok(())
503548
}
549+
550+
/// Convert a single byte to its one-character UTF-8 string form for the wire.
551+
/// Local copy of `datafusion-proto`'s `byte_to_string`; kept byte-identical.
552+
#[cfg(feature = "proto")]
553+
fn proto_byte_to_string(b: u8, description: &str) -> Result<String> {
554+
let bytes = &[b];
555+
let s = std::str::from_utf8(bytes).map_err(|_| {
556+
datafusion_common::internal_datafusion_err!(
557+
"Invalid CSV {description}: can not represent {bytes:0x?} as utf8"
558+
)
559+
})?;
560+
Ok(s.to_owned())
561+
}
562+
563+
/// Convert a one-character string from the wire back to a single byte.
564+
/// Local copy of `datafusion-proto`'s `str_to_byte`; kept byte-identical.
565+
#[cfg(feature = "proto")]
566+
fn proto_str_to_byte(s: &str, description: &str) -> Result<u8> {
567+
datafusion_common::assert_eq_or_internal_err!(
568+
s.len(),
569+
1,
570+
"Invalid CSV {description}: expected single character, got {s}"
571+
);
572+
Ok(s.as_bytes()[0])
573+
}
574+
575+
#[cfg(feature = "proto")]
576+
impl CsvSource {
577+
/// Reconstruct a `DataSourceExec` (wrapping a `FileScanConfig` over a
578+
/// `CsvSource`) from a `CsvScan` [`PhysicalPlanNode`].
579+
///
580+
/// The inverse of [`FileSource::try_to_proto`] on `CsvSource`; kept
581+
/// byte-compatible with the former `try_into_csv_scan_physical_plan` in
582+
/// `datafusion-proto`.
583+
///
584+
/// [`PhysicalPlanNode`]: datafusion_proto_models::protobuf::PhysicalPlanNode
585+
pub fn try_from_proto(
586+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
587+
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
588+
) -> Result<Arc<dyn ExecutionPlan>> {
589+
use datafusion_common::config::CsvOptions;
590+
use datafusion_datasource::file_compression_type::FileCompressionType;
591+
use datafusion_datasource::file_scan_config::{
592+
FileScanConfig, FileScanConfigBuilder,
593+
};
594+
use datafusion_datasource::source::DataSourceExec;
595+
use datafusion_proto_models::protobuf;
596+
597+
let scan = match &node.physical_plan_type {
598+
Some(protobuf::physical_plan_node::PhysicalPlanType::CsvScan(scan)) => scan,
599+
_ => {
600+
return datafusion_common::internal_err!(
601+
"PhysicalPlanNode is not a CsvScan"
602+
);
603+
}
604+
};
605+
606+
let base_conf = scan.base_conf.as_ref().ok_or_else(|| {
607+
datafusion_common::internal_datafusion_err!(
608+
"CsvScanExecNode is missing required field 'base_conf'"
609+
)
610+
})?;
611+
612+
let escape = match &scan.optional_escape {
613+
Some(protobuf::csv_scan_exec_node::OptionalEscape::Escape(escape)) => {
614+
Some(proto_str_to_byte(escape, "escape")?)
615+
}
616+
None => None,
617+
};
618+
let comment = match &scan.optional_comment {
619+
Some(protobuf::csv_scan_exec_node::OptionalComment::Comment(comment)) => {
620+
Some(proto_str_to_byte(comment, "comment")?)
621+
}
622+
None => None,
623+
};
624+
625+
// Parse table schema with partition columns.
626+
let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?;
627+
628+
let csv_options = CsvOptions {
629+
has_header: Some(scan.has_header),
630+
delimiter: proto_str_to_byte(&scan.delimiter, "delimiter")?,
631+
quote: proto_str_to_byte(&scan.quote, "quote")?,
632+
newlines_in_values: Some(scan.newlines_in_values),
633+
..Default::default()
634+
};
635+
let source = Arc::new(
636+
CsvSource::new(table_schema)
637+
.with_csv_options(csv_options)
638+
.with_escape(escape)
639+
.with_comment(comment),
640+
);
641+
642+
let conf = FileScanConfigBuilder::from(FileScanConfig::from_proto_conf(
643+
base_conf, ctx, source,
644+
)?)
645+
.with_file_compression_type(FileCompressionType::UNCOMPRESSED)
646+
.build();
647+
Ok(DataSourceExec::from_data_source(conf))
648+
}
649+
}

‎datafusion/datasource/Cargo.toml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,14 @@ all-features = true
3434
backtrace = ["datafusion-common/backtrace"]
3535
compression = ["async-compression", "liblzma", "bzip2", "flate2", "zstd", "tokio-util"]
3636
default = ["compression"]
37+
# Enables `DataSource::try_to_proto` / `FileSource::try_to_proto` serialization
38+
# hooks and the shared `FileScanConfig` <-> proto conversion. Off by default so
39+
# consumers that never serialize plans pay nothing. Mirrors the `proto` feature
40+
# on `datafusion-physical-plan`.
41+
proto = [
42+
"dep:datafusion-proto-models",
43+
"datafusion-physical-plan/proto",
44+
]
3745

3846
[dependencies]
3947
arrow = { workspace = true }
@@ -56,6 +64,7 @@ datafusion-physical-expr = { workspace = true }
5664
datafusion-physical-expr-adapter = { workspace = true }
5765
datafusion-physical-expr-common = { workspace = true }
5866
datafusion-physical-plan = { workspace = true }
67+
datafusion-proto-models = { workspace = true, optional = true }
5968
datafusion-session = { workspace = true }
6069
flate2 = { workspace = true, optional = true }
6170
futures = { workspace = true }

‎datafusion/datasource/src/file.rs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -351,6 +351,31 @@ pub trait FileSource: Any + Send + Sync {
351351
fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {
352352
None
353353
}
354+
355+
/// Serialize this file source into a full [`PhysicalPlanNode`] (a
356+
/// `DataSourceExec` wrapping the `FileScanConfig`), if it knows how.
357+
///
358+
/// `base` is the shared [`FileScanConfig`] this source is wrapped in; the
359+
/// format-agnostic parts (file groups, schema, statistics, ordering,
360+
/// projection, …) are encoded via
361+
/// [`FileScanConfig::to_proto_conf`](crate::file_scan_config::FileScanConfig::to_proto_conf),
362+
/// and the concrete source appends its format-specific fields (e.g. CSV
363+
/// delimiter/quote) around it.
364+
///
365+
/// * `Ok(None)` (the default) — this source has no proto hook yet; the
366+
/// caller falls back to the central downcast chain in `datafusion-proto`.
367+
/// * `Ok(Some(node))` — fully serialized; the caller must not fall back.
368+
///
369+
/// [`PhysicalPlanNode`]: datafusion_proto_models::protobuf::PhysicalPlanNode
370+
/// [`FileScanConfig`]: crate::file_scan_config::FileScanConfig
371+
#[cfg(feature = "proto")]
372+
fn try_to_proto(
373+
&self,
374+
_base: &FileScanConfig,
375+
_ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
376+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
377+
Ok(None)
378+
}
354379
}
355380

356381
impl dyn FileSource {

‎datafusion/datasource/src/file_scan_config/mod.rs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,12 @@
2020
2121
pub(crate) mod sort_pushdown;
2222

23+
/// Shared `FileScanConfig` <-> proto conversion, gated on the `proto` feature.
24+
/// Attaches inherent `to_proto_conf` / `from_proto_conf` / `parse_table_schema_from_proto`
25+
/// helpers to [`FileScanConfig`] used by every file source's `try_to_proto` hook.
26+
#[cfg(feature = "proto")]
27+
mod proto;
28+
2329
use crate::file_groups::FileGroup;
2430
use crate::{
2531
PartitionedFile, display::FileGroupsDisplay, file::FileSource,
@@ -1167,6 +1173,18 @@ impl DataSource for FileScanConfig {
11671173

11681174
Some(Arc::new(SharedWorkSource::from_config(self)) as Arc<dyn Any + Send + Sync>)
11691175
}
1176+
1177+
/// Serialize this file scan by delegating to the concrete
1178+
/// [`FileSource`](crate::file::FileSource)'s
1179+
/// [`try_to_proto`](crate::file::FileSource::try_to_proto) hook, passing
1180+
/// `self` as the shared spine it needs to emit the base config.
1181+
#[cfg(feature = "proto")]
1182+
fn try_to_proto(
1183+
&self,
1184+
ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
1185+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
1186+
self.file_source().try_to_proto(self, ctx)
1187+
}
11701188
}
11711189

11721190
impl FileScanConfig {

0 commit comments

Comments
 (0)