Skip to content

Commit 3449851

Browse files
buraksennkosiew
authored andcommitted
refactor(proto): migrate ParquetSource serde (apache#24169)
## Which issue does this PR close? - Closes apache#23517. ## Rationale for this change Part of epic apache#23494. Moves `ParquetSource` protobuf serialization from central dispatch into the source implementation. ## What changes are included in this PR? Add protobuf serialization and deserialization to `ParquetSource`, including the pushdown predicate and `TableParquetOptions`. Rebuild the cached Parquet reader factory from the runtime environment during deserialization. ## Are these changes tested? Yes. Existing Parquet round-trip tests cover this change. ## Are there any user-facing changes? no
1 parent dc4638a commit 3449851

3 files changed

Lines changed: 153 additions & 95 deletions

File tree

‎datafusion/datasource-parquet/src/source.rs‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1057,6 +1057,135 @@ impl FileSource for ParquetSource {
10571057
inner: Arc::new(new_source) as Arc<dyn FileSource>,
10581058
})
10591059
}
1060+
1061+
/// Emit a `ParquetScan` node wrapping the shared base config plus the
1062+
/// Parquet-specific pushdown predicate and `TableParquetOptions`.
1063+
#[cfg(feature = "proto")]
1064+
fn try_to_proto(
1065+
&self,
1066+
base: &FileScanConfig,
1067+
ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
1068+
) -> datafusion_common::Result<
1069+
Option<datafusion_proto_models::protobuf::PhysicalPlanNode>,
1070+
> {
1071+
use datafusion_proto_models::protobuf;
1072+
use protobuf::physical_plan_node::PhysicalPlanType;
1073+
1074+
let predicate = self
1075+
.filter()
1076+
.map(|pred| ctx.encode_expr(&pred))
1077+
.transpose()?;
1078+
1079+
let node = protobuf::ParquetScanExecNode {
1080+
base_conf: Some(base.try_to_proto(ctx)?),
1081+
predicate,
1082+
parquet_options: Some(self.table_parquet_options().try_into()?),
1083+
};
1084+
Ok(Some(protobuf::PhysicalPlanNode {
1085+
physical_plan_type: Some(PhysicalPlanType::ParquetScan(node)),
1086+
}))
1087+
}
1088+
}
1089+
1090+
#[cfg(feature = "proto")]
1091+
impl ParquetSource {
1092+
/// Reconstructs a `DataSourceExec` from a protobuf `ParquetScan`.
1093+
///
1094+
/// Rebuilds the reader factory from the decode context because it is not serialized.
1095+
pub fn try_from_proto(
1096+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
1097+
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
1098+
) -> datafusion_common::Result<Arc<dyn datafusion_physical_plan::ExecutionPlan>> {
1099+
use crate::CachedParquetFileReaderFactory;
1100+
use arrow::datatypes::Schema;
1101+
use datafusion_common::config::TableParquetOptions;
1102+
use datafusion_datasource::file_scan_config::FileScanConfig;
1103+
use datafusion_datasource::source::DataSourceExec;
1104+
use datafusion_execution::object_store::ObjectStoreUrl;
1105+
use datafusion_proto_models::protobuf;
1106+
1107+
let scan = match &node.physical_plan_type {
1108+
Some(protobuf::physical_plan_node::PhysicalPlanType::ParquetScan(scan)) => {
1109+
scan
1110+
}
1111+
_ => {
1112+
return datafusion_common::internal_err!(
1113+
"PhysicalPlanNode is not a ParquetScan"
1114+
);
1115+
}
1116+
};
1117+
1118+
let base_conf = scan.base_conf.as_ref().ok_or_else(|| {
1119+
datafusion_common::internal_datafusion_err!(
1120+
"ParquetScanExecNode is missing required field 'base_conf'"
1121+
)
1122+
})?;
1123+
1124+
let schema: Arc<Schema> = Arc::new(
1125+
base_conf
1126+
.schema
1127+
.as_ref()
1128+
.ok_or_else(|| {
1129+
datafusion_common::internal_datafusion_err!(
1130+
"FileScanExecConf is missing required field 'schema'"
1131+
)
1132+
})?
1133+
.try_into()?,
1134+
);
1135+
1136+
// The predicate was serialized against the scan's output schema, so it
1137+
// must be decoded against the projected schema when a projection is
1138+
// present.
1139+
let predicate_schema = if !base_conf.projection.is_empty() {
1140+
let projected_fields: Vec<_> = base_conf
1141+
.projection
1142+
.iter()
1143+
.map(|&i| schema.field(i as usize).clone())
1144+
.collect();
1145+
Arc::new(Schema::new(projected_fields))
1146+
} else {
1147+
schema
1148+
};
1149+
1150+
let predicate = scan
1151+
.predicate
1152+
.as_ref()
1153+
.map(|expr| ctx.decode_expr(expr, predicate_schema.as_ref()))
1154+
.transpose()?;
1155+
1156+
let mut options = TableParquetOptions::default();
1157+
if let Some(table_options) = scan.parquet_options.as_ref() {
1158+
options = table_options.try_into()?;
1159+
}
1160+
1161+
let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?;
1162+
let object_store_url = match base_conf.object_store_url.is_empty() {
1163+
false => ObjectStoreUrl::parse(&base_conf.object_store_url)?,
1164+
true => ObjectStoreUrl::local_filesystem(),
1165+
};
1166+
let store = ctx
1167+
.task_ctx()
1168+
.runtime_env()
1169+
.object_store(object_store_url)?;
1170+
let metadata_cache = ctx
1171+
.task_ctx()
1172+
.runtime_env()
1173+
.cache_manager
1174+
.get_file_metadata_cache();
1175+
let reader_factory =
1176+
Arc::new(CachedParquetFileReaderFactory::new(store, metadata_cache));
1177+
1178+
let mut source = ParquetSource::new(table_schema)
1179+
.with_parquet_file_reader_factory(reader_factory)
1180+
.with_table_parquet_options(options);
1181+
1182+
if let Some(predicate) = predicate {
1183+
source = source.with_predicate(predicate);
1184+
}
1185+
let base_config =
1186+
FileScanConfig::try_from_proto(base_conf, ctx, Arc::new(source))?;
1187+
Ok(DataSourceExec::from_data_source(base_config))
1188+
}
10601189
}
10611190

10621191
/// Returns the a [`TableSchema`] containing a [`RowNumber`] virtual column and a [`Column`] expression referencing its row index column.

‎datafusion/proto/src/physical_plan/from_proto.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -425,6 +425,10 @@ pub fn parse_protobuf_partitioning(
425425
.transpose()
426426
.map(Option::flatten)
427427
}
428+
#[deprecated(
429+
since = "55.0.0",
430+
note = "unused by DataFusion; use `FileScanConfig::parse_table_schema_from_proto` to reconstruct the full table schema"
431+
)]
428432
pub fn parse_protobuf_file_scan_schema(
429433
proto: &protobuf::FileScanExecConf,
430434
) -> Result<Arc<Schema>> {

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 20 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,6 @@ use datafusion_catalog::memory::MemorySourceConfig;
2626
use datafusion_common::{
2727
DataFusionError, Result, internal_datafusion_err, internal_err, not_impl_err,
2828
};
29-
#[cfg(feature = "parquet")]
30-
use datafusion_datasource::file::FileSource;
3129
use datafusion_datasource::file_scan_config::FileScanConfig;
3230
use datafusion_datasource::sink::DataSinkExec;
3331
use datafusion_datasource::source::{DataSource, DataSourceExec};
@@ -39,13 +37,9 @@ use datafusion_datasource_csv::source::CsvSource;
3937
use datafusion_datasource_json::file_format::JsonSink;
4038
use datafusion_datasource_json::source::JsonSource;
4139
#[cfg(feature = "parquet")]
42-
use datafusion_datasource_parquet::CachedParquetFileReaderFactory;
43-
#[cfg(feature = "parquet")]
4440
use datafusion_datasource_parquet::file_format::ParquetSink;
4541
#[cfg(feature = "parquet")]
4642
use datafusion_datasource_parquet::source::ParquetSource;
47-
#[cfg(feature = "parquet")]
48-
use datafusion_execution::object_store::ObjectStoreUrl;
4943
use datafusion_execution::{FunctionRegistry, TaskContext};
5044
use datafusion_expr::physical_planning_context::ScalarSubqueryResults;
5145
use datafusion_expr::{AggregateUDF, HigherOrderUDF, ScalarUDF, WindowUDF};
@@ -1088,8 +1082,15 @@ pub trait PhysicalPlanNodeExt: Sized {
10881082
PhysicalPlanType::JsonScan(scan) => {
10891083
self.try_into_json_scan_physical_plan(scan, ctx, proto_converter)
10901084
}
1091-
PhysicalPlanType::ParquetScan(scan) => {
1092-
self.try_into_parquet_scan_physical_plan(scan, ctx, proto_converter)
1085+
PhysicalPlanType::ParquetScan(_) => {
1086+
#[cfg(feature = "parquet")]
1087+
{
1088+
ParquetSource::try_from_proto(self.node(), &decode_ctx)
1089+
}
1090+
#[cfg(not(feature = "parquet"))]
1091+
panic!(
1092+
"Unable to process a Parquet PhysicalPlan when `parquet` feature is not enabled"
1093+
)
10931094
}
10941095
PhysicalPlanType::AvroScan(scan) => {
10951096
self.try_into_avro_scan_physical_plan(scan, ctx, proto_converter)
@@ -1396,6 +1397,10 @@ pub trait PhysicalPlanNodeExt: Sized {
13961397
}
13971398

13981399
#[cfg_attr(not(feature = "parquet"), expect(unused_variables))]
1400+
#[deprecated(
1401+
since = "55.0.0",
1402+
note = "unused by DataFusion; `ParquetSource` deserializes itself via `ParquetSource::try_from_proto`"
1403+
)]
13991404
fn try_into_parquet_scan_physical_plan(
14001405
&self,
14011406
scan: &protobuf::ParquetScanExecNode,
@@ -1404,74 +1409,17 @@ pub trait PhysicalPlanNodeExt: Sized {
14041409
) -> Result<Arc<dyn ExecutionPlan>> {
14051410
#[cfg(feature = "parquet")]
14061411
{
1407-
let schema = from_proto::parse_protobuf_file_scan_schema(
1408-
scan.base_conf.as_ref().unwrap(),
1409-
)?;
1410-
1411-
// Check if there's a projection and use projected schema for predicate parsing
1412-
let base_conf = scan.base_conf.as_ref().unwrap();
1413-
let predicate_schema = if !base_conf.projection.is_empty() {
1414-
// Create projected schema for parsing the predicate
1415-
let projected_fields: Vec<_> = base_conf
1416-
.projection
1417-
.iter()
1418-
.map(|&i| schema.field(i as usize).clone())
1419-
.collect();
1420-
Arc::new(Schema::new(projected_fields))
1421-
} else {
1422-
schema
1423-
};
1424-
1425-
let predicate = scan
1426-
.predicate
1427-
.as_ref()
1428-
.map(|expr| {
1429-
proto_converter.proto_to_physical_expr(
1430-
expr,
1431-
predicate_schema.as_ref(),
1432-
ctx,
1433-
)
1434-
})
1435-
.transpose()?;
1436-
let mut options = datafusion_common::config::TableParquetOptions::default();
1437-
1438-
if let Some(table_options) = scan.parquet_options.as_ref() {
1439-
options = table_options.try_into()?;
1440-
}
1441-
1442-
// Parse table schema with partition columns
1443-
let table_schema = parse_table_schema_from_proto(base_conf)?;
1444-
let object_store_url = match base_conf.object_store_url.is_empty() {
1445-
false => ObjectStoreUrl::parse(&base_conf.object_store_url)?,
1446-
true => ObjectStoreUrl::local_filesystem(),
1412+
let node = protobuf::PhysicalPlanNode {
1413+
physical_plan_type: Some(PhysicalPlanType::ParquetScan(scan.clone())),
14471414
};
1448-
let store = ctx
1449-
.task_ctx()
1450-
.runtime_env()
1451-
.object_store(object_store_url)?;
1452-
let metadata_cache = ctx
1453-
.task_ctx()
1454-
.runtime_env()
1455-
.cache_manager
1456-
.get_file_metadata_cache();
1457-
let reader_factory =
1458-
Arc::new(CachedParquetFileReaderFactory::new(store, metadata_cache));
1459-
1460-
let mut source = ParquetSource::new(table_schema)
1461-
.with_parquet_file_reader_factory(reader_factory)
1462-
.with_table_parquet_options(options);
1463-
1464-
if let Some(predicate) = predicate {
1465-
source = source.with_predicate(predicate);
1466-
}
1467-
let base_config = parse_protobuf_file_scan_config(
1468-
base_conf,
1415+
let decoder = ConverterPlanDecoder {
14691416
ctx,
14701417
proto_converter,
1471-
Arc::new(source),
1472-
)?;
1473-
Ok(DataSourceExec::from_data_source(base_config))
1418+
};
1419+
let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
1420+
ParquetSource::try_from_proto(&node, &decode_ctx)
14741421
}
1422+
14751423
#[cfg(not(feature = "parquet"))]
14761424
panic!(
14771425
"Unable to process a Parquet PhysicalPlan when `parquet` feature is not enabled"
@@ -2562,29 +2510,6 @@ pub trait PhysicalPlanNodeExt: Sized {
25622510
}
25632511
}
25642512

2565-
#[cfg(feature = "parquet")]
2566-
if let Some((maybe_parquet, conf)) =
2567-
data_source_exec.downcast_to_file_source::<ParquetSource>()
2568-
{
2569-
let predicate = conf
2570-
.filter()
2571-
.map(|pred| proto_converter.physical_expr_to_proto(&pred, codec))
2572-
.transpose()?;
2573-
return Ok(Some(protobuf::PhysicalPlanNode {
2574-
physical_plan_type: Some(PhysicalPlanType::ParquetScan(
2575-
protobuf::ParquetScanExecNode {
2576-
base_conf: Some(serialize_file_scan_config(
2577-
maybe_parquet,
2578-
codec,
2579-
proto_converter,
2580-
)?),
2581-
predicate,
2582-
parquet_options: Some(conf.table_parquet_options().try_into()?),
2583-
},
2584-
)),
2585-
}));
2586-
}
2587-
25882513
#[cfg(feature = "avro")]
25892514
if let Some(maybe_avro) = data_source.downcast_ref::<FileScanConfig>() {
25902515
let source = maybe_avro.file_source();

0 commit comments

Comments
 (0)