Skip to content

Commit 8649d17

Browse files
jayshrivastavakosiew
authored andcommitted
add ExecutionPlan::dynamic_expressions_produced() method (apache#24068)
## Which issue does this PR close? - Closes: apache#23814 - Closes: datafusion-contrib/datafusion-distributed#584 ## Rationale for this change External users who wish to propagate dynamic filter updates across network boundaries need to know in which direction updates need to flow. Thus `ExecutionPlan::apply_expressions` is not enough. The proposal in apache#23814 is to provide a separate API for dynamic filter producers. ## What changes are included in this PR? This change adds a new method `ExecutionPlan::dynamic_expressions_produced(&self) -> Vec<Arc<dyn PhysicalExpr>>` which should return dynamic filters produced by the `ExecutionPlan`. ## Are these changes tested? Yes. Adds a new invariant that all expressions returned by `ExecutionPlan::dynamic_expressions_produced` have an expression id. ## Are there any user-facing changes? There's a new API, `ExecutionPlan::dynamic_expressions_produced(&self) -> Vec<Arc<dyn PhysicalExpr>>`
1 parent 291c1cc commit 8649d17

10 files changed

Lines changed: 380 additions & 57 deletions

File tree

‎datafusion/core/tests/physical_optimizer/filter_pushdown.rs‎

Lines changed: 78 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,11 @@ use datafusion::{
3434
scalar::ScalarValue,
3535
};
3636
use datafusion_catalog::memory::DataSourceExec;
37-
use datafusion_common::config::ConfigOptions;
37+
use datafusion_common::{
38+
JoinType,
39+
config::ConfigOptions,
40+
tree_node::{TreeNode, TreeNodeRecursion},
41+
};
3842
use datafusion_datasource::{
3943
PartitionedFile, file_groups::FileGroup, file_scan_config::FileScanConfigBuilder,
4044
};
@@ -60,6 +64,7 @@ use datafusion_physical_plan::{
6064
coalesce_partitions::CoalescePartitionsExec,
6165
collect,
6266
filter::{FilterExec, FilterExecBuilder},
67+
joins::{HashJoinExec, PartitionMode},
6368
projection::ProjectionExec,
6469
repartition::RepartitionExec,
6570
sorts::sort::SortExec,
@@ -2914,12 +2919,16 @@ async fn test_hashjoin_hash_table_pushdown_collect_left() {
29142919
);
29152920
}
29162921

2917-
// Not portable to sqllogictest: asserts on `HashJoinExec::dynamic_filter_for_test().is_used()`
2918-
// which is a debug-only API. The observable behavior (probe-side scan
2922+
// Not portable to sqllogictest: asserts on the dynamic filter's Arc ownership.
2923+
// The observable behavior (probe-side scan
29192924
// receiving the dynamic filter when the data source supports it) is
29202925
// already covered by the simpler CollectLeft port in push_down_filter_parquet.slt;
29212926
// the with_support(false) branch has no SQL analog (parquet always supports
29222927
// pushdown).
2928+
#[expect(
2929+
deprecated,
2930+
reason = "the borrowed getter avoids adding a producer Arc that is_used would count"
2931+
)]
29232932
#[tokio::test]
29242933
async fn test_hashjoin_dynamic_filter_pushdown_is_used() {
29252934
use datafusion_common::JoinType;
@@ -3101,6 +3110,72 @@ async fn test_filter_with_projection_pushdown() {
31013110
assert_batches_eq!(expected, &result);
31023111
}
31033112

3113+
#[test]
3114+
fn test_discover_dynamic_expression_producers() {
3115+
fn producer_count(plan: &Arc<dyn ExecutionPlan>) -> usize {
3116+
let mut count = 0;
3117+
plan.apply(|node| {
3118+
count += node.dynamic_expressions_produced().len();
3119+
Ok(TreeNodeRecursion::Continue)
3120+
})
3121+
.expect("plan traversal should succeed");
3122+
count
3123+
}
3124+
3125+
let build_schema = Arc::new(Schema::new(vec![
3126+
Field::new("a", DataType::Utf8, false),
3127+
Field::new("b", DataType::Int32, false),
3128+
]));
3129+
let build_scan = TestScanBuilder::new(Arc::clone(&build_schema))
3130+
.with_support(true)
3131+
.with_batches(vec![
3132+
record_batch!(("a", Utf8, ["foo", "bar"]), ("b", Int32, [1, 2])).unwrap(),
3133+
])
3134+
.build();
3135+
3136+
let probe_schema = Arc::new(Schema::new(vec![
3137+
Field::new("a", DataType::Utf8, false),
3138+
Field::new("c", DataType::Float64, false),
3139+
]));
3140+
let probe_scan = TestScanBuilder::new(Arc::clone(&probe_schema))
3141+
.with_support(true)
3142+
.with_batches(vec![
3143+
record_batch!(
3144+
("a", Utf8, ["foo", "bar", "baz", "qux"]),
3145+
("c", Float64, [1.0, 2.0, 3.0, 4.0])
3146+
)
3147+
.unwrap(),
3148+
])
3149+
.build();
3150+
3151+
let plan = Arc::new(
3152+
HashJoinExec::try_new(
3153+
build_scan,
3154+
probe_scan,
3155+
vec![(
3156+
col("a", &build_schema).unwrap(),
3157+
col("a", &probe_schema).unwrap(),
3158+
)],
3159+
None,
3160+
&JoinType::Inner,
3161+
None,
3162+
PartitionMode::CollectLeft,
3163+
datafusion_common::NullEquality::NullEqualsNothing,
3164+
false,
3165+
)
3166+
.unwrap(),
3167+
) as Arc<dyn ExecutionPlan>;
3168+
assert_eq!(producer_count(&plan), 0);
3169+
3170+
let mut config = ConfigOptions::default();
3171+
config.optimizer.enable_dynamic_filter_pushdown = true;
3172+
config.execution.parquet.pushdown_filters = true;
3173+
let optimized_plan = FilterPushdown::new_post_optimization()
3174+
.optimize(plan, &config)
3175+
.unwrap();
3176+
assert_eq!(producer_count(&optimized_plan), 1);
3177+
}
3178+
31043179
// ==== Filter pushdown through SortExec tests ====
31053180

31063181
/// FilterExec above a plain SortExec (no fetch) should be pushed below it.

‎datafusion/ffi/src/execution_plan.rs‎

Lines changed: 72 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ use tokio::runtime::Handle;
3333

3434
use crate::config::FFI_ConfigOptions;
3535
use crate::execution::FFI_TaskContext;
36+
use crate::physical_expr::FFI_PhysicalExpr;
3637
use crate::physical_expr::metrics::FFI_MetricsSet;
3738
use crate::plan_properties::FFI_PlanProperties;
3839
use crate::record_batch_stream::FFI_RecordBatchStream;
@@ -50,6 +51,10 @@ pub struct FFI_ExecutionPlan {
5051
/// Return a vector of children plans
5152
pub children: unsafe extern "C" fn(plan: &Self) -> SVec<FFI_ExecutionPlan>,
5253

54+
/// Return the dynamic expressions produced by this plan node.
55+
pub dynamic_expressions_produced:
56+
unsafe extern "C" fn(plan: &Self) -> SVec<FFI_PhysicalExpr>,
57+
5358
pub with_new_children:
5459
unsafe extern "C" fn(plan: &Self, children: SVec<Self>) -> FFI_Result<Self>,
5560

@@ -138,6 +143,16 @@ unsafe extern "C" fn children_fn_wrapper(
138143
.collect()
139144
}
140145

146+
unsafe extern "C" fn dynamic_expressions_produced_fn_wrapper(
147+
plan: &FFI_ExecutionPlan,
148+
) -> SVec<FFI_PhysicalExpr> {
149+
plan.inner()
150+
.dynamic_expressions_produced()
151+
.into_iter()
152+
.map(FFI_PhysicalExpr::from)
153+
.collect()
154+
}
155+
141156
unsafe extern "C" fn with_new_children_fn_wrapper(
142157
plan: &FFI_ExecutionPlan,
143158
children: SVec<FFI_ExecutionPlan>,
@@ -306,6 +321,7 @@ impl FFI_ExecutionPlan {
306321
Self {
307322
properties: properties_fn_wrapper,
308323
children: children_fn_wrapper,
324+
dynamic_expressions_produced: dynamic_expressions_produced_fn_wrapper,
309325
with_new_children: with_new_children_fn_wrapper,
310326
name: name_fn_wrapper,
311327
execute: execute_fn_wrapper,
@@ -442,6 +458,15 @@ impl ExecutionPlan for ForeignExecutionPlan {
442458
}
443459
}
444460

461+
fn dynamic_expressions_produced(
462+
&self,
463+
) -> Vec<Arc<dyn datafusion_physical_plan::PhysicalExpr>> {
464+
unsafe { (self.plan.dynamic_expressions_produced)(&self.plan) }
465+
.iter()
466+
.map(<Arc<dyn datafusion_physical_plan::PhysicalExpr>>::from)
467+
.collect()
468+
}
469+
445470
fn repartitioned(
446471
&self,
447472
target_partitions: usize,
@@ -474,15 +499,17 @@ impl ExecutionPlan for ForeignExecutionPlan {
474499

475500
#[cfg(any(test, feature = "integration-tests"))]
476501
pub mod tests {
477-
use datafusion_physical_plan::Partitioning;
502+
use datafusion_physical_expr::expressions::{DynamicFilterPhysicalExpr, lit};
478503
use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};
504+
use datafusion_physical_plan::{Partitioning, PhysicalExpr};
479505

480506
use super::*;
481507

482508
#[derive(Debug)]
483509
pub struct EmptyExec {
484510
props: Arc<PlanProperties>,
485511
children: Vec<Arc<dyn ExecutionPlan>>,
512+
dynamic_expressions: Vec<Arc<dyn PhysicalExpr>>,
486513
metrics: Option<MetricsSet>,
487514
statistics: Option<Statistics>,
488515
}
@@ -497,6 +524,7 @@ pub mod tests {
497524
Boundedness::Bounded,
498525
)),
499526
children: Vec::default(),
527+
dynamic_expressions: Vec::default(),
500528
metrics: None,
501529
statistics: None,
502530
}
@@ -511,6 +539,14 @@ pub mod tests {
511539
self.statistics = Some(statistics);
512540
self
513541
}
542+
543+
pub fn with_dynamic_expressions(
544+
mut self,
545+
dynamic_expressions: Vec<Arc<dyn PhysicalExpr>>,
546+
) -> Self {
547+
self.dynamic_expressions = dynamic_expressions;
548+
self
549+
}
514550
}
515551

516552
impl DisplayAs for EmptyExec {
@@ -543,6 +579,7 @@ pub mod tests {
543579
Ok(Arc::new(EmptyExec {
544580
props: Arc::clone(&self.props),
545581
children,
582+
dynamic_expressions: self.dynamic_expressions.clone(),
546583
metrics: self.metrics.clone(),
547584
statistics: self.statistics.clone(),
548585
}))
@@ -556,6 +593,10 @@ pub mod tests {
556593
unimplemented!()
557594
}
558595

596+
fn dynamic_expressions_produced(&self) -> Vec<Arc<dyn PhysicalExpr>> {
597+
self.dynamic_expressions.iter().map(Arc::clone).collect()
598+
}
599+
559600
fn metrics(&self) -> Option<MetricsSet> {
560601
self.metrics.clone()
561602
}
@@ -571,6 +612,10 @@ pub mod tests {
571612
}
572613
}
573614

615+
pub(crate) fn create_dynamic_filter() -> Arc<DynamicFilterPhysicalExpr> {
616+
Arc::new(DynamicFilterPhysicalExpr::new(vec![], lit(true)))
617+
}
618+
574619
#[test]
575620
fn test_round_trip_ffi_execution_plan() -> Result<()> {
576621
let schema = Arc::new(arrow::datatypes::Schema::new(vec![
@@ -600,6 +645,32 @@ pub mod tests {
600645
Ok(())
601646
}
602647

648+
#[test]
649+
fn test_ffi_execution_plan_dynamic_expressions_produced() -> Result<()> {
650+
let schema = Arc::new(arrow::datatypes::Schema::empty());
651+
let dynamic_filter = create_dynamic_filter();
652+
let expected_id = dynamic_filter
653+
.expression_id()
654+
.expect("dynamic filters always have an expression ID");
655+
let expression: Arc<dyn PhysicalExpr> = Arc::clone(&dynamic_filter) as _;
656+
let original_plan =
657+
Arc::new(EmptyExec::new(schema).with_dynamic_expressions(vec![expression]));
658+
659+
let mut ffi_plan = FFI_ExecutionPlan::new(original_plan, None);
660+
ffi_plan.library_marker_id = crate::mock_foreign_marker_id;
661+
let foreign_plan: Arc<dyn ExecutionPlan> = (&ffi_plan).try_into()?;
662+
foreign_plan.check_invariants(
663+
datafusion_physical_plan::execution_plan::InvariantLevel::Always,
664+
)?;
665+
666+
let produced = foreign_plan.dynamic_expressions_produced();
667+
assert_eq!(produced.len(), 1);
668+
assert_eq!(produced[0].expression_id(), Some(expected_id));
669+
drop(foreign_plan);
670+
assert_eq!(produced[0].expression_id(), Some(expected_id));
671+
Ok(())
672+
}
673+
603674
#[test]
604675
fn test_ffi_execution_plan_children() -> Result<()> {
605676
let schema = Arc::new(arrow::datatypes::Schema::new(vec![

‎datafusion/ffi/src/physical_expr/mod.rs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,8 @@ pub struct FFI_PhysicalExpr {
120120

121121
pub is_volatile_node: unsafe extern "C" fn(&Self) -> bool,
122122

123+
pub expression_id: unsafe extern "C" fn(&Self) -> FFI_Option<u64>,
124+
123125
// Display trait
124126
pub display: unsafe extern "C" fn(&Self) -> SString,
125127

@@ -387,6 +389,13 @@ unsafe extern "C" fn is_volatile_node_fn_wrapper(expr: &FFI_PhysicalExpr) -> boo
387389
let expr = expr.inner();
388390
expr.is_volatile_node()
389391
}
392+
393+
unsafe extern "C" fn expression_id_fn_wrapper(
394+
expr: &FFI_PhysicalExpr,
395+
) -> FFI_Option<u64> {
396+
expr.inner().expression_id().into()
397+
}
398+
390399
unsafe extern "C" fn display_fn_wrapper(expr: &FFI_PhysicalExpr) -> SString {
391400
let expr = expr.inner();
392401
format!("{expr}").into()
@@ -434,6 +443,7 @@ unsafe extern "C" fn clone_fn_wrapper(expr: &FFI_PhysicalExpr) -> FFI_PhysicalEx
434443
snapshot: snapshot_fn_wrapper,
435444
snapshot_generation: snapshot_generation_fn_wrapper,
436445
is_volatile_node: is_volatile_node_fn_wrapper,
446+
expression_id: expression_id_fn_wrapper,
437447
display: display_fn_wrapper,
438448
hash: hash_fn_wrapper,
439449
clone: clone_fn_wrapper,
@@ -477,6 +487,7 @@ impl From<Arc<dyn PhysicalExpr>> for FFI_PhysicalExpr {
477487
snapshot: snapshot_fn_wrapper,
478488
snapshot_generation: snapshot_generation_fn_wrapper,
479489
is_volatile_node: is_volatile_node_fn_wrapper,
490+
expression_id: expression_id_fn_wrapper,
480491
display: display_fn_wrapper,
481492
hash: hash_fn_wrapper,
482493
clone: clone_fn_wrapper,
@@ -713,6 +724,10 @@ impl PhysicalExpr for ForeignPhysicalExpr {
713724
fn is_volatile_node(&self) -> bool {
714725
unsafe { (self.expr.is_volatile_node)(&self.expr) }
715726
}
727+
728+
fn expression_id(&self) -> Option<u64> {
729+
unsafe { (self.expr.expression_id)(&self.expr).into() }
730+
}
716731
}
717732

718733
impl Eq for ForeignPhysicalExpr {}

‎datafusion/ffi/src/tests/mod.rs‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ use datafusion_common::stats::Precision;
2828
use datafusion_common::{ColumnStatistics, Statistics};
2929
use datafusion_common::{Result, ScalarValue};
3030
use datafusion_expr::{Expr, TableType};
31+
use datafusion_physical_expr::PhysicalExpr;
3132
use datafusion_physical_plan::ExecutionPlan;
3233
use sync_provider::create_sync_table_provider;
3334
use udf_udaf_udwf::{
@@ -40,7 +41,7 @@ use crate::catalog_provider::FFI_CatalogProvider;
4041
use crate::catalog_provider_list::FFI_CatalogProviderList;
4142
use crate::config::extension_options::FFI_ExtensionOptions;
4243
use crate::execution_plan::FFI_ExecutionPlan;
43-
use crate::execution_plan::tests::EmptyExec;
44+
use crate::execution_plan::tests::{EmptyExec, create_dynamic_filter};
4445
use crate::physical_optimizer::FFI_PhysicalOptimizerRule;
4546
use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec;
4647
use crate::proto::physical_extension_codec::FFI_PhysicalExtensionCodec;
@@ -112,6 +113,8 @@ pub struct ForeignLibraryModule {
112113

113114
pub create_empty_exec: extern "C" fn() -> FFI_ExecutionPlan,
114115

116+
pub create_exec_with_dynamic_expressions: extern "C" fn() -> FFI_ExecutionPlan,
117+
115118
pub create_exec_with_statistics: extern "C" fn() -> FFI_ExecutionPlan,
116119

117120
pub create_table_with_statistics:
@@ -177,6 +180,14 @@ pub(crate) extern "C" fn create_empty_exec() -> FFI_ExecutionPlan {
177180
FFI_ExecutionPlan::new(plan, None)
178181
}
179182

183+
pub(crate) extern "C" fn create_exec_with_dynamic_expressions() -> FFI_ExecutionPlan {
184+
let schema = Arc::new(Schema::empty());
185+
let expression: Arc<dyn PhysicalExpr> = create_dynamic_filter();
186+
let plan =
187+
Arc::new(EmptyExec::new(schema).with_dynamic_expressions(vec![expression]));
188+
FFI_ExecutionPlan::new(plan, None)
189+
}
190+
180191
/// Returns canonical statistics used by both the producer and consumer sides of
181192
/// the integration tests so round-trips can be asserted without hard-coding
182193
/// the values in two places.
@@ -275,6 +286,7 @@ pub extern "C" fn datafusion_ffi_get_module() -> ForeignLibraryModule {
275286
create_rank_udwf: create_ffi_rank_func,
276287
create_extension_options: config::create_extension_options,
277288
create_empty_exec,
289+
create_exec_with_dynamic_expressions,
278290
create_exec_with_statistics,
279291
create_table_with_statistics,
280292
create_physical_optimizer_rule:

0 commit comments

Comments
 (0)