Skip to content

Commit a43b336

Browse files
buraksennkosiew
authored andcommitted
refactor(proto): migrate JsonSource serde (apache#24178)
## Which issue does this PR close? - Part of apache#23516. ## Rationale for this change Part of epic apache#23494. Moves `JsonSource` protobuf serialization from the central dispatch into the source implementation. ## What changes are included in this PR? Add protobuf serialization and deserialization to `JsonSource`. Repoint the live decode arm to `JsonSource::try_from_proto` and remove the old central encode arm. Keep `try_into_json_scan_physical_plan` as a deprecated compatibility wrapper that delegates to the new implementation. The protobuf wire format remains unchanged. ## Are these changes tested? Yes. Added `roundtrip_json_scan`. ## Are there any user-facing changes? The existing `PhysicalPlanNodeExt` method remains available as a deprecated compatibility wrapper. There is no immediate API removal or wire-format change.
1 parent 3449851 commit a43b336

4 files changed

Lines changed: 85 additions & 29 deletions

File tree

‎datafusion/datasource-json/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ version.workspace = true
3131
all-features = true
3232

3333
[features]
34+
# Enables protobuf serialization hooks for JSON sources and sinks.
3435
proto = [
3536
"dep:datafusion-proto-models",
3637
"datafusion-datasource/proto",

‎datafusion/datasource-json/src/source.rs‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,60 @@ impl FileSource for JsonSource {
231231
fn file_type(&self) -> &str {
232232
"json"
233233
}
234+
235+
/// Emit a `JsonScan` node wrapping the shared base config.
236+
#[cfg(feature = "proto")]
237+
fn try_to_proto(
238+
&self,
239+
base: &FileScanConfig,
240+
ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
241+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
242+
use datafusion_proto_models::protobuf;
243+
use protobuf::physical_plan_node::PhysicalPlanType;
244+
245+
let node = protobuf::JsonScanExecNode {
246+
base_conf: Some(base.try_to_proto(ctx)?),
247+
};
248+
Ok(Some(protobuf::PhysicalPlanNode {
249+
physical_plan_type: Some(PhysicalPlanType::JsonScan(node)),
250+
}))
251+
}
252+
}
253+
254+
#[cfg(feature = "proto")]
255+
impl JsonSource {
256+
/// Reconstructs a `DataSourceExec` from a protobuf `JsonScan`.
257+
///
258+
/// Defaults to newline-delimited JSON because protobuf does not encode the mode.
259+
pub fn try_from_proto(
260+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
261+
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
262+
) -> Result<Arc<dyn ExecutionPlan>> {
263+
use datafusion_datasource::file_scan_config::FileScanConfig;
264+
use datafusion_datasource::source::DataSourceExec;
265+
use datafusion_proto_models::protobuf;
266+
267+
let scan = match &node.physical_plan_type {
268+
Some(protobuf::physical_plan_node::PhysicalPlanType::JsonScan(scan)) => scan,
269+
_ => {
270+
return datafusion_common::internal_err!(
271+
"PhysicalPlanNode is not a JsonScan"
272+
);
273+
}
274+
};
275+
276+
let base_conf = scan.base_conf.as_ref().ok_or_else(|| {
277+
datafusion_common::internal_datafusion_err!(
278+
"JsonScanExecNode is missing required field 'base_conf'"
279+
)
280+
})?;
281+
282+
let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?;
283+
let source = Arc::new(JsonSource::new(table_schema));
284+
285+
let conf = FileScanConfig::try_from_proto(base_conf, ctx, source)?;
286+
Ok(DataSourceExec::from_data_source(conf))
287+
}
234288
}
235289

236290
impl FileOpener for JsonOpener {

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 13 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1079,8 +1079,8 @@ pub trait PhysicalPlanNodeExt: Sized {
10791079
PhysicalPlanType::CsvScan(_) => {
10801080
CsvSource::try_from_proto(self.node(), &decode_ctx)
10811081
}
1082-
PhysicalPlanType::JsonScan(scan) => {
1083-
self.try_into_json_scan_physical_plan(scan, ctx, proto_converter)
1082+
PhysicalPlanType::JsonScan(_) => {
1083+
JsonSource::try_from_proto(self.node(), &decode_ctx)
10841084
}
10851085
PhysicalPlanType::ParquetScan(_) => {
10861086
#[cfg(feature = "parquet")]
@@ -1360,21 +1360,25 @@ pub trait PhysicalPlanNodeExt: Sized {
13601360
CsvSource::try_from_proto(&node, &decode_ctx)
13611361
}
13621362

1363+
#[deprecated(
1364+
since = "55.0.0",
1365+
note = "unused by DataFusion; `JsonSource` deserializes itself via `JsonSource::try_from_proto`"
1366+
)]
13631367
fn try_into_json_scan_physical_plan(
13641368
&self,
13651369
scan: &protobuf::JsonScanExecNode,
13661370
ctx: &PhysicalPlanDecodeContext<'_>,
13671371
proto_converter: &dyn PhysicalProtoConverterExtension,
13681372
) -> Result<Arc<dyn ExecutionPlan>> {
1369-
let base_conf = scan.base_conf.as_ref().unwrap();
1370-
let table_schema = parse_table_schema_from_proto(base_conf)?;
1371-
let scan_conf = parse_protobuf_file_scan_config(
1372-
base_conf,
1373+
let node = protobuf::PhysicalPlanNode {
1374+
physical_plan_type: Some(PhysicalPlanType::JsonScan(scan.clone())),
1375+
};
1376+
let decoder = ConverterPlanDecoder {
13731377
ctx,
13741378
proto_converter,
1375-
Arc::new(JsonSource::new(table_schema)),
1376-
)?;
1377-
Ok(DataSourceExec::from_data_source(scan_conf))
1379+
};
1380+
let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
1381+
JsonSource::try_from_proto(&node, &decode_ctx)
13781382
}
13791383

13801384
fn try_into_arrow_scan_physical_plan(
@@ -2476,23 +2480,6 @@ pub trait PhysicalPlanNodeExt: Sized {
24762480
) -> Result<Option<protobuf::PhysicalPlanNode>> {
24772481
let data_source = data_source_exec.data_source();
24782482

2479-
if let Some(scan_conf) = data_source.downcast_ref::<FileScanConfig>() {
2480-
let source = scan_conf.file_source();
2481-
if let Some(_json_source) = source.downcast_ref::<JsonSource>() {
2482-
return Ok(Some(protobuf::PhysicalPlanNode {
2483-
physical_plan_type: Some(PhysicalPlanType::JsonScan(
2484-
protobuf::JsonScanExecNode {
2485-
base_conf: Some(serialize_file_scan_config(
2486-
scan_conf,
2487-
codec,
2488-
proto_converter,
2489-
)?),
2490-
},
2491-
)),
2492-
}));
2493-
}
2494-
}
2495-
24962483
if let Some(scan_conf) = data_source.downcast_ref::<FileScanConfig>() {
24972484
let source = scan_conf.file_source();
24982485
if let Some(_arrow_source) = source.downcast_ref::<ArrowSource>() {

‎datafusion/proto/tests/cases/roundtrip_physical_plan.rs‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@ use datafusion::datasource::listing::{
3737
use datafusion::datasource::object_store::ObjectStoreUrl;
3838
use datafusion::datasource::physical_plan::{
3939
ArrowSource, CsvSource, FileGroup, FileOutputMode, FileScanConfig,
40-
FileScanConfigBuilder, FileSinkConfig, ParquetSource, wrap_partition_type_in_dict,
41-
wrap_partition_value_in_dict,
40+
FileScanConfigBuilder, FileSinkConfig, JsonSource, ParquetSource,
41+
wrap_partition_type_in_dict, wrap_partition_value_in_dict,
4242
};
4343
use datafusion::datasource::sink::{DataSink, DataSinkExec};
4444
use datafusion::datasource::source::DataSourceExec;
@@ -1363,6 +1363,21 @@ fn roundtrip_arrow_scan() -> Result<()> {
13631363
roundtrip_test(DataSourceExec::from_data_source(scan_config))
13641364
}
13651365

1366+
#[test]
1367+
fn roundtrip_json_scan() -> Result<()> {
1368+
let file_schema =
1369+
Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
1370+
let file_source = Arc::new(JsonSource::new(TableSchema::from(&file_schema)));
1371+
let scan_config =
1372+
FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
1373+
.with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
1374+
"/path/to/file.json".to_string(),
1375+
1024,
1376+
)])])
1377+
.build();
1378+
roundtrip_test(DataSourceExec::from_data_source(scan_config))
1379+
}
1380+
13661381
#[test]
13671382
fn roundtrip_csv_scan_preserves_format_options() -> Result<()> {
13681383
use datafusion::common::config::CsvOptions;
@@ -1418,7 +1433,6 @@ fn roundtrip_csv_scan_preserves_format_options() -> Result<()> {
14181433
assert!(csv_source.truncate_rows());
14191434
Ok(())
14201435
}
1421-
14221436
#[tokio::test]
14231437
async fn roundtrip_parquet_exec_with_table_partition_cols() -> Result<()> {
14241438
let mut file_group =

0 commit comments

Comments
 (0)