Skip to content

Commit 653f415

Browse files
committed
fix: pass filters pushed earlier to supports_filter_pushdown
Part of apache#19929 Let "t" be a table provider that supports exactly any single filter but not a conjunction. Consider the following optimizer pipeline: 1. Try to push `a = 1, b = 1`. `supports_filters_pushdown` returns [Exact, Inexact] OK: the optimizer records that a = 1 is pushed and creates a filter node for b = 1. ... Another optimization iteration. 2. Try to push `b = 1`. `supports_filters_pushdown` returns [Exact]. Of course, the table provider can't remember all previously pushed filters, so it has no choice but to answer `Exact`. Now, the optimizer thinks the conjunction a = 1 AND b = 1 is supported exactly, but it is not. To prevent this problem, this patch passes filters that were already pushed into the scan earlier to `supports_filters_pushdown`.
1 parent 86cb815 commit 653f415

2 files changed

Lines changed: 83 additions & 12 deletions

File tree

‎datafusion/optimizer/src/push_down_filter.rs‎

Lines changed: 82 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1130,7 +1130,12 @@ impl OptimizerRule for PushDownFilter {
11301130
}
11311131
LogicalPlan::Join(join) => push_down_join(join, Some(&filter.predicate)),
11321132
LogicalPlan::TableScan(scan) => {
1133-
let filter_predicates = split_conjunction(&filter.predicate);
1133+
let filter_predicates: Vec<_> = split_conjunction(&filter.predicate)
1134+
.into_iter()
1135+
// Add already pushed filters.
1136+
.chain(scan.filters.iter())
1137+
.unique()
1138+
.collect();
11341139

11351140
let (volatile_filters, non_volatile_filters): (Vec<&Expr>, Vec<&Expr>) =
11361141
filter_predicates
@@ -1158,13 +1163,8 @@ impl OptimizerRule for PushDownFilter {
11581163
.map(|(pred, _)| pred);
11591164

11601165
// Add new scan filters
1161-
let new_scan_filters: Vec<Expr> = scan
1162-
.filters
1163-
.iter()
1164-
.chain(new_scan_filters)
1165-
.unique()
1166-
.cloned()
1167-
.collect();
1166+
let new_scan_filters: Vec<Expr> =
1167+
new_scan_filters.unique().cloned().collect();
11681168

11691169
// Compose predicates to be of `Unsupported` or `Inexact` pushdown type, and also include volatile filters
11701170
let new_predicate: Vec<Expr> = zip
@@ -1467,8 +1467,8 @@ mod tests {
14671467
use datafusion_expr::{
14681468
ColumnarValue, ExprFunctionExt, Extension, LogicalPlanBuilder,
14691469
ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, TableSource, TableType,
1470-
UserDefinedLogicalNodeCore, Volatility, WindowFunctionDefinition, col, in_list,
1471-
in_subquery, lit,
1470+
UserDefinedLogicalNodeCore, Volatility, WindowFunctionDefinition, binary_expr,
1471+
col, in_list, in_subquery, lit,
14721472
};
14731473

14741474
use crate::OptimizerContext;
@@ -1489,7 +1489,17 @@ mod tests {
14891489
$plan:expr,
14901490
@ $expected:literal $(,)?
14911491
) => {{
1492-
let optimizer_ctx = OptimizerContext::new().with_max_passes(1);
1492+
assert_optimized_plan_equal_with_pases_num!($plan, 1, @ $expected,)
1493+
}};
1494+
}
1495+
1496+
macro_rules! assert_optimized_plan_equal_with_pases_num {
1497+
(
1498+
$plan:expr,
1499+
$max_pases: expr,
1500+
@ $expected:literal $(,)?
1501+
) => {{
1502+
let optimizer_ctx = OptimizerContext::new().with_max_passes($max_pases);
14931503
let rules: Vec<Arc<dyn crate::OptimizerRule + Send + Sync>> = vec![Arc::new(PushDownFilter::new())];
14941504
assert_optimized_plan_eq_snapshot!(
14951505
optimizer_ctx,
@@ -4315,4 +4325,65 @@ mod tests {
43154325
"
43164326
)
43174327
}
4328+
4329+
struct SingleFilterSupportSource {
4330+
schema: SchemaRef,
4331+
}
4332+
4333+
#[async_trait]
4334+
impl TableSource for SingleFilterSupportSource {
4335+
fn schema(&self) -> SchemaRef {
4336+
Arc::clone(&self.schema)
4337+
}
4338+
4339+
fn table_type(&self) -> TableType {
4340+
TableType::Base
4341+
}
4342+
4343+
fn supports_filters_pushdown(
4344+
&self,
4345+
filters: &[&Expr],
4346+
) -> Result<Vec<TableProviderFilterPushDown>> {
4347+
// Support exactly any single filter.
4348+
let mut res = vec![TableProviderFilterPushDown::Unsupported; filters.len()];
4349+
res[0] = TableProviderFilterPushDown::Exact;
4350+
Ok(res)
4351+
}
4352+
4353+
fn as_any(&self) -> &dyn Any {
4354+
self
4355+
}
4356+
}
4357+
4358+
#[test]
4359+
fn test_pushed_filters_passed_again() -> Result<()> {
4360+
let schema = Arc::new(Schema::new(vec![
4361+
Field::new("a", DataType::Int32, true),
4362+
Field::new("b", DataType::Int32, true),
4363+
]));
4364+
4365+
let plan = LogicalPlanBuilder::scan(
4366+
"t",
4367+
Arc::new(SingleFilterSupportSource {
4368+
schema: Arc::clone(&schema),
4369+
}),
4370+
None,
4371+
)?
4372+
.filter(binary_expr(
4373+
binary_expr(col("a"), Operator::Eq, lit(1)),
4374+
Operator::And,
4375+
binary_expr(col("b"), Operator::Eq, lit(4)),
4376+
))?
4377+
.build()?;
4378+
4379+
// Verify that the only one filter is pushed.
4380+
assert_optimized_plan_equal_with_pases_num!(
4381+
plan,
4382+
5,
4383+
@r"
4384+
Filter: t.b = Int32(4)
4385+
TableScan: t, full_filters=[t.a = Int32(1)]
4386+
"
4387+
)
4388+
}
43184389
}

‎datafusion/sqllogictest/test_files/predicates.slt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -666,7 +666,7 @@ logical_plan
666666
03)----Filter: lineitem.l_quantity >= Decimal128(Some(100),15,2) AND lineitem.l_quantity <= Decimal128(Some(1100),15,2) OR lineitem.l_quantity >= Decimal128(Some(1000),15,2) AND lineitem.l_quantity <= Decimal128(Some(2000),15,2) OR lineitem.l_quantity >= Decimal128(Some(2000),15,2) AND lineitem.l_quantity <= Decimal128(Some(3000),15,2)
667667
04)------TableScan: lineitem projection=[l_partkey, l_quantity], partial_filters=[lineitem.l_quantity >= Decimal128(Some(100),15,2) AND lineitem.l_quantity <= Decimal128(Some(1100),15,2) OR lineitem.l_quantity >= Decimal128(Some(1000),15,2) AND lineitem.l_quantity <= Decimal128(Some(2000),15,2) OR lineitem.l_quantity >= Decimal128(Some(2000),15,2) AND lineitem.l_quantity <= Decimal128(Some(3000),15,2)]
668668
05)----Filter: (part.p_brand = Utf8View("Brand#12") AND part.p_size <= Int32(5) OR part.p_brand = Utf8View("Brand#23") AND part.p_size <= Int32(10) OR part.p_brand = Utf8View("Brand#34") AND part.p_size <= Int32(15)) AND part.p_size >= Int32(1)
669-
06)------TableScan: part projection=[p_partkey, p_brand, p_size], partial_filters=[part.p_size >= Int32(1), part.p_brand = Utf8View("Brand#12") AND part.p_size <= Int32(5) OR part.p_brand = Utf8View("Brand#23") AND part.p_size <= Int32(10) OR part.p_brand = Utf8View("Brand#34") AND part.p_size <= Int32(15)]
669+
06)------TableScan: part projection=[p_partkey, p_brand, p_size], partial_filters=[part.p_brand = Utf8View("Brand#12") AND part.p_size <= Int32(5) OR part.p_brand = Utf8View("Brand#23") AND part.p_size <= Int32(10) OR part.p_brand = Utf8View("Brand#34") AND part.p_size <= Int32(15), part.p_size >= Int32(1)]
670670
physical_plan
671671
01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_partkey@0, p_partkey@0)], filter=p_brand@1 = Brand#12 AND l_quantity@0 >= Some(100),15,2 AND l_quantity@0 <= Some(1100),15,2 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND l_quantity@0 >= Some(1000),15,2 AND l_quantity@0 <= Some(2000),15,2 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND l_quantity@0 >= Some(2000),15,2 AND l_quantity@0 <= Some(3000),15,2 AND p_size@2 <= 15, projection=[l_partkey@0]
672672
02)--RepartitionExec: partitioning=Hash([l_partkey@0], 4), input_partitions=4

0 commit comments

Comments
 (0)