Skip to content

Commit 918013e

Browse files
authored
refactor(proto): migrate AvroSource serde (#24190)
## Which issue does this PR close? - Closes #23516. ## Rationale for this change Part of epic #23494. Moves `AvroSource` protobuf serialization from the central dispatch into the source implementation. ## What changes are included in this PR? Add protobuf serialization and deserialization to `AvroSource` and add the required `proto` feature wiring to `datafusion-datasource-avro`. Repoint the feature-gated live decode arm to `AvroSource::try_from_proto` and remove the old central encode arm. Keep `try_into_avro_scan_physical_plan` as a deprecated compatibility wrapper that delegates to the new implementation. The protobuf wire format remains unchanged. ## Are these changes tested? Yes. Added `roundtrip_avro_scan`. ## Are there any user-facing changes? The existing `PhysicalPlanNodeExt` method remains available as a deprecated compatibility wrapper. There is no immediate API removal or wire-format change.
1 parent ddb0250 commit 918013e

6 files changed

Lines changed: 101 additions & 28 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/datasource-avro/Cargo.toml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,15 @@ version.workspace = true
3030
[package.metadata.docs.rs]
3131
all-features = true
3232

33+
[features]
34+
# Enables `FileSource::try_to_proto` on `AvroSource` and the `AvroScan` decode
35+
# entry point. Mirrors the `proto` feature on `datafusion-datasource`.
36+
proto = [
37+
"dep:datafusion-proto-models",
38+
"datafusion-datasource/proto",
39+
"datafusion-physical-plan/proto",
40+
]
41+
3342
[dependencies]
3443
arrow = { workspace = true }
3544
arrow-avro = { workspace = true }
@@ -39,6 +48,7 @@ datafusion-common = { workspace = true, features = ["object_store"] }
3948
datafusion-datasource = { workspace = true }
4049
datafusion-physical-expr-adapter = { workspace = true }
4150
datafusion-physical-plan = { workspace = true }
51+
datafusion-proto-models = { workspace = true, optional = true }
4252
datafusion-session = { workspace = true }
4353
futures = { workspace = true }
4454
object_store = { workspace = true }

‎datafusion/datasource-avro/src/source.rs‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,57 @@ impl FileSource for AvroSource {
168168
// Avro OCF does not support safe byte-range splitting in this reader path.
169169
false
170170
}
171+
172+
/// Emit an `AvroScan` node wrapping the shared base config.
173+
#[cfg(feature = "proto")]
174+
fn try_to_proto(
175+
&self,
176+
base: &FileScanConfig,
177+
ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
178+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
179+
use datafusion_proto_models::protobuf;
180+
use protobuf::physical_plan_node::PhysicalPlanType;
181+
182+
let node = protobuf::AvroScanExecNode {
183+
base_conf: Some(base.try_to_proto(ctx)?),
184+
};
185+
Ok(Some(protobuf::PhysicalPlanNode {
186+
physical_plan_type: Some(PhysicalPlanType::AvroScan(node)),
187+
}))
188+
}
189+
}
190+
191+
#[cfg(feature = "proto")]
192+
impl AvroSource {
193+
/// Reconstructs a `DataSourceExec` from a protobuf `AvroScan`.
194+
pub fn try_from_proto(
195+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
196+
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
197+
) -> Result<Arc<dyn datafusion_physical_plan::ExecutionPlan>> {
198+
use datafusion_datasource::source::DataSourceExec;
199+
use datafusion_proto_models::protobuf;
200+
201+
let scan = match &node.physical_plan_type {
202+
Some(protobuf::physical_plan_node::PhysicalPlanType::AvroScan(scan)) => scan,
203+
_ => {
204+
return datafusion_common::internal_err!(
205+
"PhysicalPlanNode is not an AvroScan"
206+
);
207+
}
208+
};
209+
210+
let base_conf = scan.base_conf.as_ref().ok_or_else(|| {
211+
datafusion_common::internal_datafusion_err!(
212+
"AvroScanExecNode is missing required field 'base_conf'"
213+
)
214+
})?;
215+
216+
let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?;
217+
let source = Arc::new(AvroSource::new(table_schema));
218+
219+
let conf = FileScanConfig::try_from_proto(base_conf, ctx, source)?;
220+
Ok(DataSourceExec::from_data_source(conf))
221+
}
171222
}
172223

173224
mod private {

‎datafusion/proto/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ datafusion-catalog-listing = { workspace = true }
6363
datafusion-common = { workspace = true }
6464
datafusion-datasource = { workspace = true, features = ["proto"] }
6565
datafusion-datasource-arrow = { workspace = true }
66-
datafusion-datasource-avro = { workspace = true, optional = true }
66+
datafusion-datasource-avro = { workspace = true, optional = true, features = ["proto"] }
6767
datafusion-datasource-csv = { workspace = true, features = ["proto"] }
6868
datafusion-datasource-json = { workspace = true, features = ["proto"] }
6969
datafusion-datasource-parquet = { workspace = true, optional = true, features = ["proto"] }

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 20 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1090,8 +1090,15 @@ pub trait PhysicalPlanNodeExt: Sized {
10901090
"Unable to process a Parquet PhysicalPlan when `parquet` feature is not enabled"
10911091
)
10921092
}
1093-
PhysicalPlanType::AvroScan(scan) => {
1094-
self.try_into_avro_scan_physical_plan(scan, ctx, proto_converter)
1093+
PhysicalPlanType::AvroScan(_) => {
1094+
#[cfg(feature = "avro")]
1095+
{
1096+
AvroSource::try_from_proto(self.node(), &decode_ctx)
1097+
}
1098+
#[cfg(not(feature = "avro"))]
1099+
panic!(
1100+
"Unable to process a Avro PhysicalPlan when `avro` feature is not enabled"
1101+
)
10951102
}
10961103
PhysicalPlanType::MemoryScan(_) => {
10971104
MemorySourceConfig::try_from_proto(self.node(), &decode_ctx)
@@ -1429,6 +1436,10 @@ pub trait PhysicalPlanNodeExt: Sized {
14291436
}
14301437

14311438
#[cfg_attr(not(feature = "avro"), expect(unused_variables))]
1439+
#[deprecated(
1440+
since = "55.0.0",
1441+
note = "unused by DataFusion; `AvroSource` deserializes itself via `AvroSource::try_from_proto`"
1442+
)]
14321443
fn try_into_avro_scan_physical_plan(
14331444
&self,
14341445
scan: &protobuf::AvroScanExecNode,
@@ -1437,15 +1448,15 @@ pub trait PhysicalPlanNodeExt: Sized {
14371448
) -> Result<Arc<dyn ExecutionPlan>> {
14381449
#[cfg(feature = "avro")]
14391450
{
1440-
let table_schema =
1441-
parse_table_schema_from_proto(scan.base_conf.as_ref().unwrap())?;
1442-
let conf = parse_protobuf_file_scan_config(
1443-
scan.base_conf.as_ref().unwrap(),
1451+
let node = protobuf::PhysicalPlanNode {
1452+
physical_plan_type: Some(PhysicalPlanType::AvroScan(scan.clone())),
1453+
};
1454+
let decoder = ConverterPlanDecoder {
14441455
ctx,
14451456
proto_converter,
1446-
Arc::new(AvroSource::new(table_schema)),
1447-
)?;
1448-
Ok(DataSourceExec::from_data_source(conf))
1457+
};
1458+
let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
1459+
AvroSource::try_from_proto(&node, &decode_ctx)
14491460
}
14501461

14511462
#[cfg(not(feature = "avro"))]
@@ -2472,24 +2483,6 @@ pub trait PhysicalPlanNodeExt: Sized {
24722483
}
24732484
}
24742485

2475-
#[cfg(feature = "avro")]
2476-
if let Some(maybe_avro) = data_source.downcast_ref::<FileScanConfig>() {
2477-
let source = maybe_avro.file_source();
2478-
if source.downcast_ref::<AvroSource>().is_some() {
2479-
return Ok(Some(protobuf::PhysicalPlanNode {
2480-
physical_plan_type: Some(PhysicalPlanType::AvroScan(
2481-
protobuf::AvroScanExecNode {
2482-
base_conf: Some(serialize_file_scan_config(
2483-
maybe_avro,
2484-
codec,
2485-
proto_converter,
2486-
)?),
2487-
},
2488-
)),
2489-
}));
2490-
}
2491-
}
2492-
24932486
Ok(None)
24942487
}
24952488

‎datafusion/proto/tests/cases/roundtrip_physical_plan.rs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1378,6 +1378,24 @@ fn roundtrip_json_scan() -> Result<()> {
13781378
roundtrip_test(DataSourceExec::from_data_source(scan_config))
13791379
}
13801380

1381+
#[cfg(feature = "avro")]
1382+
#[test]
1383+
fn roundtrip_avro_scan() -> Result<()> {
1384+
use datafusion_datasource_avro::source::AvroSource;
1385+
1386+
let file_schema =
1387+
Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
1388+
let file_source = Arc::new(AvroSource::new(TableSchema::from(&file_schema)));
1389+
let scan_config =
1390+
FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
1391+
.with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
1392+
"/path/to/file.avro".to_string(),
1393+
1024,
1394+
)])])
1395+
.build();
1396+
roundtrip_test(DataSourceExec::from_data_source(scan_config))
1397+
}
1398+
13811399
#[test]
13821400
fn roundtrip_csv_scan_preserves_format_options() -> Result<()> {
13831401
use datafusion::common::config::CsvOptions;

0 commit comments

Comments
 (0)