Skip to content

Commit 5f60509

Browse files
adriangbclaude
andcommitted
prototype: migrate Json/Arrow/Avro/Parquet FileSources onto the proto hook (apache#22419)
Completes the file-scan family: all 5 FileSources now serialize via FileSource::try_to_proto and decode via <Source>::try_from_proto. The central scan encode branches in try_from_data_source_exec are deleted (only MemorySourceConfig remains) and the 5 scan decode arms repointed. Parquet carries its predicate via ctx.encode_expr and options via proto-common; none of the formats need the raw codec. New roundtrip_json_scan / roundtrip_avro_scan tests (+ existing csv/arrow/parquet scan tests) exercise every FileSource hook end-to-end. 192 tests pass (193 with avro). Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01SfBPC2yYCwHUn4V8pQCVj3
1 parent 18ef6cf commit 5f60509

12 files changed

Lines changed: 448 additions & 95 deletions

File tree

‎Cargo.lock‎

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/datasource-arrow/Cargo.toml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ datafusion-execution = { workspace = true }
4242
datafusion-expr = { workspace = true }
4343
datafusion-physical-expr-common = { workspace = true }
4444
datafusion-physical-plan = { workspace = true }
45+
datafusion-proto-models = { workspace = true, optional = true }
4546
datafusion-session = { workspace = true }
4647
futures = { workspace = true }
4748
itertools = { workspace = true }
@@ -65,3 +66,10 @@ path = "src/mod.rs"
6566
# This feature is deprecated, as core functionality in the SpillManager requires all features
6667
# it enabled, and will be removed in a future version.
6768
compression = []
69+
# Enables `FileSource::try_to_proto` on `ArrowSource` and the `ArrowScan` decode
70+
# entry point. Mirrors the `proto` feature on `datafusion-datasource`.
71+
proto = [
72+
"dep:datafusion-proto-models",
73+
"datafusion-datasource/proto",
74+
"datafusion-physical-plan/proto",
75+
]

‎datafusion/datasource-arrow/src/source.rs‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,67 @@ impl FileSource for ArrowSource {
392392
fn projection(&self) -> Option<&ProjectionExprs> {
393393
Some(&self.projection.source)
394394
}
395+
396+
/// Emit an `ArrowScan` node wrapping the shared base config. Arrow is
397+
/// base_conf-only (no format-specific fields). Kept byte-identical to the
398+
/// former `try_from_data_source_exec` ArrowScan branch in `datafusion-proto`.
399+
#[cfg(feature = "proto")]
400+
fn try_to_proto(
401+
&self,
402+
base: &FileScanConfig,
403+
ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
404+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
405+
use datafusion_proto_models::protobuf;
406+
use protobuf::physical_plan_node::PhysicalPlanType;
407+
408+
Ok(Some(protobuf::PhysicalPlanNode {
409+
physical_plan_type: Some(PhysicalPlanType::ArrowScan(
410+
protobuf::ArrowScanExecNode {
411+
base_conf: Some(base.to_proto_conf(ctx)?),
412+
},
413+
)),
414+
}))
415+
}
416+
}
417+
418+
#[cfg(feature = "proto")]
419+
impl ArrowSource {
420+
/// Reconstruct a `DataSourceExec` (wrapping a `FileScanConfig` over an
421+
/// `ArrowSource`) from an `ArrowScan` [`PhysicalPlanNode`].
422+
///
423+
/// The inverse of [`FileSource::try_to_proto`] on `ArrowSource`; kept
424+
/// byte-compatible with the former `try_into_arrow_scan_physical_plan` in
425+
/// `datafusion-proto`.
426+
///
427+
/// [`PhysicalPlanNode`]: datafusion_proto_models::protobuf::PhysicalPlanNode
428+
pub fn try_from_proto(
429+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
430+
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
431+
) -> Result<Arc<dyn datafusion_physical_plan::ExecutionPlan>> {
432+
use datafusion_datasource::file_scan_config::FileScanConfig;
433+
use datafusion_datasource::source::DataSourceExec;
434+
use datafusion_proto_models::protobuf;
435+
436+
let scan = match &node.physical_plan_type {
437+
Some(protobuf::physical_plan_node::PhysicalPlanType::ArrowScan(scan)) => scan,
438+
_ => {
439+
return datafusion_common::internal_err!(
440+
"PhysicalPlanNode is not an ArrowScan"
441+
);
442+
}
443+
};
444+
445+
let base_conf = scan.base_conf.as_ref().ok_or_else(|| {
446+
datafusion_common::internal_datafusion_err!(
447+
"base_conf in ArrowScanExecNode is missing."
448+
)
449+
})?;
450+
451+
let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?;
452+
let source = Arc::new(ArrowSource::new_file_source(table_schema));
453+
let scan_conf = FileScanConfig::from_proto_conf(base_conf, ctx, source)?;
454+
Ok(DataSourceExec::from_data_source(scan_conf))
455+
}
395456
}
396457

397458
/// `FileOpener` wrapper for both Arrow IPC file and stream formats

‎datafusion/datasource-avro/Cargo.toml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,15 @@ version.workspace = true
3030
[package.metadata.docs.rs]
3131
all-features = true
3232

33+
[features]
34+
# Enables `FileSource::try_to_proto` on `AvroSource` and the `AvroScan` decode
35+
# entry point. Mirrors the `proto` feature on `datafusion-datasource`.
36+
proto = [
37+
"dep:datafusion-proto-models",
38+
"datafusion-datasource/proto",
39+
"datafusion-physical-plan/proto",
40+
]
41+
3342
[dependencies]
3443
arrow = { workspace = true }
3544
arrow-avro = { workspace = true }
@@ -39,6 +48,7 @@ datafusion-common = { workspace = true, features = ["object_store"] }
3948
datafusion-datasource = { workspace = true }
4049
datafusion-physical-expr-adapter = { workspace = true }
4150
datafusion-physical-plan = { workspace = true }
51+
datafusion-proto-models = { workspace = true, optional = true }
4252
datafusion-session = { workspace = true }
4353
futures = { workspace = true }
4454
object_store = { workspace = true }

‎datafusion/datasource-avro/src/source.rs‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,67 @@ impl FileSource for AvroSource {
168168
// Avro OCF does not support safe byte-range splitting in this reader path.
169169
false
170170
}
171+
172+
/// Emit an `AvroScan` node wrapping the shared base config. Avro carries no
173+
/// format-specific fields, so this is just the base conf. Kept
174+
/// byte-identical to the former `try_from_data_source_exec` AvroScan branch
175+
/// in `datafusion-proto`.
176+
#[cfg(feature = "proto")]
177+
fn try_to_proto(
178+
&self,
179+
base: &FileScanConfig,
180+
ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
181+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
182+
use datafusion_proto_models::protobuf;
183+
use protobuf::physical_plan_node::PhysicalPlanType;
184+
185+
let node = protobuf::AvroScanExecNode {
186+
base_conf: Some(base.to_proto_conf(ctx)?),
187+
};
188+
Ok(Some(protobuf::PhysicalPlanNode {
189+
physical_plan_type: Some(PhysicalPlanType::AvroScan(node)),
190+
}))
191+
}
192+
}
193+
194+
#[cfg(feature = "proto")]
195+
impl AvroSource {
196+
/// Reconstruct a `DataSourceExec` (wrapping a `FileScanConfig` over an
197+
/// `AvroSource`) from an `AvroScan`
198+
/// [`PhysicalPlanNode`](datafusion_proto_models::protobuf::PhysicalPlanNode).
199+
///
200+
/// The inverse of [`FileSource::try_to_proto`] on `AvroSource`; kept
201+
/// byte-compatible with the former `try_into_avro_scan_physical_plan` in
202+
/// `datafusion-proto`.
203+
pub fn try_from_proto(
204+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
205+
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
206+
) -> Result<Arc<dyn datafusion_physical_plan::ExecutionPlan>> {
207+
use datafusion_datasource::source::DataSourceExec;
208+
use datafusion_proto_models::protobuf;
209+
210+
let scan = match &node.physical_plan_type {
211+
Some(protobuf::physical_plan_node::PhysicalPlanType::AvroScan(scan)) => scan,
212+
_ => {
213+
return datafusion_common::internal_err!(
214+
"PhysicalPlanNode is not an AvroScan"
215+
);
216+
}
217+
};
218+
219+
let base_conf = scan.base_conf.as_ref().ok_or_else(|| {
220+
datafusion_common::internal_datafusion_err!(
221+
"AvroScanExecNode is missing required field 'base_conf'"
222+
)
223+
})?;
224+
225+
// Parse table schema with partition columns, then rebuild the source.
226+
let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?;
227+
let source = Arc::new(AvroSource::new(table_schema));
228+
229+
let conf = FileScanConfig::from_proto_conf(base_conf, ctx, source)?;
230+
Ok(DataSourceExec::from_data_source(conf))
231+
}
171232
}
172233

173234
mod private {

‎datafusion/datasource-json/Cargo.toml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,15 @@ version.workspace = true
3030
[package.metadata.docs.rs]
3131
all-features = true
3232

33+
[features]
34+
# Enables `FileSource::try_to_proto` on `JsonSource` and the `JsonScan` decode
35+
# entry point. Mirrors the `proto` feature on `datafusion-datasource`.
36+
proto = [
37+
"dep:datafusion-proto-models",
38+
"datafusion-datasource/proto",
39+
"datafusion-physical-plan/proto",
40+
]
41+
3342
[dependencies]
3443
arrow = { workspace = true }
3544
async-trait = { workspace = true }
@@ -41,6 +50,7 @@ datafusion-execution = { workspace = true }
4150
datafusion-expr = { workspace = true }
4251
datafusion-physical-expr-common = { workspace = true }
4352
datafusion-physical-plan = { workspace = true }
53+
datafusion-proto-models = { workspace = true, optional = true }
4454
datafusion-session = { workspace = true }
4555
futures = { workspace = true }
4656
object_store = { workspace = true }

‎datafusion/datasource-json/src/source.rs‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,75 @@ impl FileSource for JsonSource {
231231
fn file_type(&self) -> &str {
232232
"json"
233233
}
234+
235+
/// Emit a `JsonScan` node wrapping the shared base config. JSON has no
236+
/// format-specific fields, so the node is `base_conf` only. Kept
237+
/// byte-identical to the former `try_from_data_source_exec` JsonScan branch
238+
/// in `datafusion-proto`.
239+
#[cfg(feature = "proto")]
240+
fn try_to_proto(
241+
&self,
242+
base: &FileScanConfig,
243+
ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
244+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
245+
use datafusion_proto_models::protobuf;
246+
use protobuf::physical_plan_node::PhysicalPlanType;
247+
248+
let node = protobuf::JsonScanExecNode {
249+
base_conf: Some(base.to_proto_conf(ctx)?),
250+
};
251+
Ok(Some(protobuf::PhysicalPlanNode {
252+
physical_plan_type: Some(PhysicalPlanType::JsonScan(node)),
253+
}))
254+
}
255+
}
256+
257+
#[cfg(feature = "proto")]
258+
impl JsonSource {
259+
/// Reconstruct a `DataSourceExec` (wrapping a `FileScanConfig` over a
260+
/// `JsonSource`) from a `JsonScan` [`PhysicalPlanNode`].
261+
///
262+
/// The inverse of [`FileSource::try_to_proto`] on `JsonSource`; kept
263+
/// byte-compatible with the former `try_into_json_scan_physical_plan` in
264+
/// `datafusion-proto`.
265+
///
266+
/// [`PhysicalPlanNode`]: datafusion_proto_models::protobuf::PhysicalPlanNode
267+
pub fn try_from_proto(
268+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
269+
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
270+
) -> Result<Arc<dyn ExecutionPlan>> {
271+
use datafusion_datasource::file_scan_config::{
272+
FileScanConfig, FileScanConfigBuilder,
273+
};
274+
use datafusion_datasource::source::DataSourceExec;
275+
use datafusion_proto_models::protobuf;
276+
277+
let scan = match &node.physical_plan_type {
278+
Some(protobuf::physical_plan_node::PhysicalPlanType::JsonScan(scan)) => scan,
279+
_ => {
280+
return datafusion_common::internal_err!(
281+
"PhysicalPlanNode is not a JsonScan"
282+
);
283+
}
284+
};
285+
286+
let base_conf = scan.base_conf.as_ref().ok_or_else(|| {
287+
datafusion_common::internal_datafusion_err!(
288+
"JsonScanExecNode is missing required field 'base_conf'"
289+
)
290+
})?;
291+
292+
// Parse table schema with partition columns.
293+
let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?;
294+
295+
let source = Arc::new(JsonSource::new(table_schema));
296+
297+
let conf = FileScanConfigBuilder::from(FileScanConfig::from_proto_conf(
298+
base_conf, ctx, source,
299+
)?)
300+
.build();
301+
Ok(DataSourceExec::from_data_source(conf))
302+
}
234303
}
235304

236305
impl FileOpener for JsonOpener {

‎datafusion/datasource-parquet/Cargo.toml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ datafusion-physical-expr = { workspace = true }
4646
datafusion-physical-expr-adapter = { workspace = true }
4747
datafusion-physical-expr-common = { workspace = true }
4848
datafusion-physical-plan = { workspace = true }
49+
datafusion-proto-models = { workspace = true, optional = true }
4950
datafusion-pruning = { workspace = true }
5051
datafusion-session = { workspace = true }
5152
futures = { workspace = true }
@@ -74,6 +75,13 @@ name = "datafusion_datasource_parquet"
7475
path = "src/mod.rs"
7576

7677
[features]
78+
# Enables `FileSource::try_to_proto` on `ParquetSource` and the `ParquetScan`
79+
# decode entry point. Mirrors the `proto` feature on `datafusion-datasource-csv`.
80+
proto = [
81+
"dep:datafusion-proto-models",
82+
"datafusion-datasource/proto",
83+
"datafusion-physical-plan/proto",
84+
]
7785
parquet_encryption = [
7886
"parquet/encryption",
7987
"datafusion-common/parquet_encryption",

0 commit comments

Comments
 (0)