Skip to content

Commit e2e8e1c

Browse files
Add DataSource/FileSource proto hooks and FileScanConfig serde (apache#23683)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> - Closes apache#23497. ## Rationale for this change File scan serialization currently depends on central type downcasts in `datafusion-proto`. This PR adds the foundation needed to move each data source to its own protobuf hooks while preserving the existing wire format and fallback behavior. It unblocks the concrete source migrations tracked by apache#23516, apache#23517, and apache#23518. <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. --> ## What changes are included in this PR? - Add feature-gated `try_to_proto` hooks to `DataSource` and `FileSource`. - Delegate serialization from `DataSourceExec` through `FileScanConfig` to its concrete `FileSource`. - Add shared `FileScanConfig` protobuf encoding and decoding in the datasource crate. - Keep the existing central serializer as the fallback for sources that still return `Ok(None)`. - Preserve the legacy protobuf wire format. - Add compatibility test <!-- There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR. --> ## Are these changes tested? Yes <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> ## Are there any user-facing changes? This adds non-breaking, feature-gated public serialization hooks for data source and file source implementations. <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> <!-- If there are any breaking changes to public APIs, please add the `api change` label. --> --------- Co-authored-by: Adrian Garcia Badaracco <[email protected]>
1 parent 09dd8d2 commit e2e8e1c

8 files changed

Lines changed: 879 additions & 183 deletions

File tree

‎datafusion/datasource/Cargo.toml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,10 @@ all-features = true
3434
backtrace = ["datafusion-common/backtrace"]
3535
compression = ["async-compression", "liblzma", "bzip2", "flate2", "zstd", "tokio-util"]
3636
default = ["compression"]
37-
# Enables protobuf conversions for datasource types and serialization hooks.
38-
# Off by default so consumers that never serialize plans pay nothing.
37+
# Enables protobuf conversions for datasource types, source serialization hooks,
38+
# and the shared `FileScanConfig` <-> proto conversion. Off by default so
39+
# consumers that never serialize plans pay nothing. Mirrors the `proto` feature
40+
# on `datafusion-physical-plan`.
3941
proto = [
4042
"dep:datafusion-proto-models",
4143
"datafusion-physical-plan/proto",

‎datafusion/datasource/src/file.rs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -351,6 +351,31 @@ pub trait FileSource: Any + Send + Sync {
351351
fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {
352352
None
353353
}
354+
355+
/// Serialize this file source into a full [`PhysicalPlanNode`] (a
356+
/// `DataSourceExec` wrapping the `FileScanConfig`), if it knows how.
357+
///
358+
/// `base` is the shared [`FileScanConfig`] this source is wrapped in; the
359+
/// format-agnostic parts (file groups, schema, statistics, ordering,
360+
/// projection, …) are encoded via
361+
/// [`FileScanConfig::try_to_proto`](crate::file_scan_config::FileScanConfig::try_to_proto),
362+
/// and the concrete source appends its format-specific fields (e.g. CSV
363+
/// delimiter/quote) around it.
364+
///
365+
/// * `Ok(None)` (the default) — this source has no proto hook yet; the
366+
/// caller falls back to the central downcast chain in `datafusion-proto`.
367+
/// * `Ok(Some(node))` — fully serialized; the caller must not fall back.
368+
///
369+
/// [`PhysicalPlanNode`]: datafusion_proto_models::protobuf::PhysicalPlanNode
370+
/// [`FileScanConfig`]: crate::file_scan_config::FileScanConfig
371+
#[cfg(feature = "proto")]
372+
fn try_to_proto(
373+
&self,
374+
_base: &FileScanConfig,
375+
_ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
376+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
377+
Ok(None)
378+
}
354379
}
355380

356381
impl dyn FileSource {

‎datafusion/datasource/src/file_scan_config/mod.rs‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,13 @@
2020
2121
pub(crate) mod sort_pushdown;
2222

23+
/// Shared `FileScanConfig` <-> proto conversion, gated on the `proto` feature.
24+
/// Attaches inherent `try_to_proto` / `try_from_proto` /
25+
/// `parse_table_schema_from_proto` helpers to [`FileScanConfig`] used by every
26+
/// file source's `try_to_proto` hook.
27+
#[cfg(feature = "proto")]
28+
mod proto;
29+
2330
use crate::file_groups::FileGroup;
2431
use crate::{
2532
PartitionedFile, display::FileGroupsDisplay, file::FileSource,
@@ -1176,6 +1183,18 @@ impl DataSource for FileScanConfig {
11761183

11771184
Some(Arc::new(SharedWorkSource::from_config(self)) as Arc<dyn Any + Send + Sync>)
11781185
}
1186+
1187+
/// Serialize this file scan by delegating to the concrete
1188+
/// [`FileSource`]'s
1189+
/// [`try_to_proto`](crate::file::FileSource::try_to_proto) hook, passing
1190+
/// `self` as the shared spine it needs to emit the base config.
1191+
#[cfg(feature = "proto")]
1192+
fn try_to_proto(
1193+
&self,
1194+
ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
1195+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
1196+
self.file_source().try_to_proto(self, ctx)
1197+
}
11791198
}
11801199

11811200
impl FileScanConfig {
@@ -1569,12 +1588,18 @@ mod tests {
15691588
use datafusion_common::{Result, assert_batches_eq, internal_err};
15701589
use datafusion_execution::TaskContext;
15711590
use datafusion_expr::SortExpr;
1591+
#[cfg(feature = "proto")]
1592+
use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};
15721593
use datafusion_physical_expr::create_physical_sort_expr;
15731594
use datafusion_physical_expr::expressions::Literal;
15741595
use datafusion_physical_expr::projection::ProjectionExpr;
15751596
use datafusion_physical_expr::projection::ProjectionExprs;
15761597
use datafusion_physical_plan::ExecutionPlan;
15771598
use datafusion_physical_plan::execution_plan::collect;
1599+
#[cfg(feature = "proto")]
1600+
use datafusion_physical_plan::proto::{ExecutionPlanEncode, ExecutionPlanEncodeCtx};
1601+
#[cfg(feature = "proto")]
1602+
use datafusion_proto_models::protobuf::{PhysicalExprNode, PhysicalPlanNode};
15781603
use futures::FutureExt as _;
15791604
use futures::StreamExt as _;
15801605
use futures::stream;
@@ -1633,6 +1658,108 @@ mod tests {
16331658
}
16341659
}
16351660

1661+
#[cfg(feature = "proto")]
1662+
#[derive(Clone)]
1663+
struct ProtoHookSource {
1664+
metrics: ExecutionPlanMetricsSet,
1665+
table_schema: TableSchema,
1666+
}
1667+
1668+
#[cfg(feature = "proto")]
1669+
impl ProtoHookSource {
1670+
fn new(table_schema: TableSchema) -> Self {
1671+
Self {
1672+
metrics: ExecutionPlanMetricsSet::new(),
1673+
table_schema,
1674+
}
1675+
}
1676+
}
1677+
1678+
#[cfg(feature = "proto")]
1679+
impl FileSource for ProtoHookSource {
1680+
fn create_file_opener(
1681+
&self,
1682+
_object_store: Arc<dyn ObjectStore>,
1683+
_base_config: &FileScanConfig,
1684+
_partition: usize,
1685+
) -> Result<Arc<dyn crate::file_stream::FileOpener>> {
1686+
internal_err!("not needed for proto delegation test")
1687+
}
1688+
1689+
fn table_schema(&self) -> &TableSchema {
1690+
&self.table_schema
1691+
}
1692+
1693+
fn with_batch_size(&self, _batch_size: usize) -> Arc<dyn FileSource> {
1694+
Arc::new(self.clone())
1695+
}
1696+
1697+
fn metrics(&self) -> &ExecutionPlanMetricsSet {
1698+
&self.metrics
1699+
}
1700+
1701+
fn file_type(&self) -> &str {
1702+
"proto-hook-test"
1703+
}
1704+
1705+
fn try_to_proto(
1706+
&self,
1707+
_base: &FileScanConfig,
1708+
_ctx: &ExecutionPlanEncodeCtx<'_>,
1709+
) -> Result<Option<PhysicalPlanNode>> {
1710+
Ok(Some(PhysicalPlanNode::default()))
1711+
}
1712+
}
1713+
1714+
#[cfg(feature = "proto")]
1715+
struct UnusedPlanEncoder;
1716+
1717+
#[cfg(feature = "proto")]
1718+
impl ExecutionPlanEncode for UnusedPlanEncoder {
1719+
fn encode_plan(
1720+
&self,
1721+
_plan: &Arc<dyn ExecutionPlan>,
1722+
) -> Result<PhysicalPlanNode> {
1723+
internal_err!("not needed for proto delegation test")
1724+
}
1725+
1726+
fn encode_expr(&self, _expr: &Arc<dyn PhysicalExpr>) -> Result<PhysicalExprNode> {
1727+
internal_err!("not needed for proto delegation test")
1728+
}
1729+
1730+
fn encode_udf(&self, _udf: &ScalarUDF) -> Result<Option<Vec<u8>>> {
1731+
internal_err!("not needed for proto delegation test")
1732+
}
1733+
1734+
fn encode_udaf(&self, _udaf: &AggregateUDF) -> Result<Option<Vec<u8>>> {
1735+
internal_err!("not needed for proto delegation test")
1736+
}
1737+
1738+
fn encode_udwf(&self, _udwf: &WindowUDF) -> Result<Option<Vec<u8>>> {
1739+
internal_err!("not needed for proto delegation test")
1740+
}
1741+
}
1742+
1743+
#[cfg(feature = "proto")]
1744+
#[test]
1745+
fn data_source_exec_delegates_proto_to_file_source() -> Result<()> {
1746+
let schema = Arc::new(Schema::new(vec![Field::new(
1747+
"value",
1748+
DataType::Int32,
1749+
false,
1750+
)]));
1751+
let source = Arc::new(ProtoHookSource::new(TableSchema::from(&schema)));
1752+
let config =
1753+
FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), source)
1754+
.build();
1755+
let exec = DataSourceExec::from_data_source(config);
1756+
let encoder = UnusedPlanEncoder;
1757+
let ctx = ExecutionPlanEncodeCtx::new(&encoder);
1758+
1759+
assert_eq!(exec.try_to_proto(&ctx)?, Some(PhysicalPlanNode::default()));
1760+
Ok(())
1761+
}
1762+
16361763
#[test]
16371764
fn physical_plan_config_no_projection_tab_cols_as_field() {
16381765
let file_schema = aggr_test_schema();

0 commit comments

Comments
 (0)