Skip to content

Commit 22981aa

Browse files
committed
feat: support pushdown alias on dynamic filter with ProjectionExec
1 parent e6049de commit 22981aa

9 files changed

Lines changed: 1039 additions & 8 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/core/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,7 @@ env_logger = { workspace = true }
175175
glob = { workspace = true }
176176
insta = { workspace = true }
177177
paste = { workspace = true }
178+
pretty_assertions = "1.0"
178179
rand = { workspace = true, features = ["small_rng"] }
179180
rand_distr = "0.5"
180181
regex = { workspace = true }

‎datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs‎

Lines changed: 229 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ use datafusion_physical_plan::{
5959
coalesce_partitions::CoalescePartitionsExec,
6060
collect,
6161
filter::FilterExec,
62+
projection::ProjectionExec,
6263
repartition::RepartitionExec,
6364
sorts::sort::SortExec,
6465
};
@@ -1813,6 +1814,234 @@ fn schema() -> SchemaRef {
18131814
Arc::clone(&TEST_SCHEMA)
18141815
}
18151816

1817+
struct ProjectionDynFilterTestCase {
1818+
schema: SchemaRef,
1819+
batches: Vec<RecordBatch>,
1820+
projection: Vec<(Arc<dyn PhysicalExpr>, String)>,
1821+
sort_expr: PhysicalSortExpr,
1822+
expected_plans: Vec<String>,
1823+
}
1824+
1825+
async fn run_projection_dyn_filter_case(case: ProjectionDynFilterTestCase) {
1826+
let ProjectionDynFilterTestCase {
1827+
schema,
1828+
batches,
1829+
projection,
1830+
sort_expr,
1831+
expected_plans,
1832+
} = case;
1833+
1834+
let scan = TestScanBuilder::new(Arc::clone(&schema))
1835+
.with_support(true)
1836+
.with_batches(batches)
1837+
.build();
1838+
1839+
let projection_exec = Arc::new(ProjectionExec::try_new(projection, scan).unwrap());
1840+
1841+
let sort = Arc::new(
1842+
SortExec::new(LexOrdering::new(vec![sort_expr]).unwrap(), projection_exec)
1843+
.with_fetch(Some(2)),
1844+
) as Arc<dyn ExecutionPlan>;
1845+
1846+
let mut config = ConfigOptions::default();
1847+
config.execution.parquet.pushdown_filters = true;
1848+
config.optimizer.enable_dynamic_filter_pushdown = true;
1849+
1850+
let optimized_plan = FilterPushdown::new_post_optimization()
1851+
.optimize(Arc::clone(&sort), &config)
1852+
.unwrap();
1853+
1854+
pretty_assertions::assert_eq!(
1855+
format_plan_for_test(&optimized_plan).trim(),
1856+
expected_plans[0].trim()
1857+
);
1858+
1859+
let config = SessionConfig::new().with_batch_size(2);
1860+
let session_ctx = SessionContext::new_with_config(config);
1861+
session_ctx.register_object_store(
1862+
ObjectStoreUrl::parse("test://").unwrap().as_ref(),
1863+
Arc::new(InMemory::new()),
1864+
);
1865+
let state = session_ctx.state();
1866+
let task_ctx = state.task_ctx();
1867+
let mut stream = optimized_plan.execute(0, Arc::clone(&task_ctx)).unwrap();
1868+
for (idx, expected_plan) in expected_plans.iter().enumerate().skip(1) {
1869+
stream.next().await.unwrap().unwrap();
1870+
let formatted_plan = format_plan_for_test(&optimized_plan);
1871+
pretty_assertions::assert_eq!(
1872+
formatted_plan.trim(),
1873+
expected_plan.trim(),
1874+
"Mismatch at iteration {}",
1875+
idx
1876+
);
1877+
}
1878+
}
1879+
1880+
#[tokio::test]
1881+
async fn test_topk_with_projection_transformation_on_dyn_filter() {
1882+
let schema = Arc::new(Schema::new(vec![
1883+
Field::new("a", DataType::Int32, false),
1884+
Field::new("b", DataType::Utf8, false),
1885+
Field::new("c", DataType::Float64, false),
1886+
]));
1887+
let simple_abc = vec![
1888+
record_batch!(
1889+
("a", Int32, [1, 2, 3]),
1890+
("b", Utf8, ["x", "y", "z"]),
1891+
("c", Float64, [1.0, 2.0, 3.0])
1892+
)
1893+
.unwrap(),
1894+
];
1895+
1896+
// Case 1: Reordering [b, a]
1897+
run_projection_dyn_filter_case(ProjectionDynFilterTestCase {
1898+
schema: Arc::clone(&schema),
1899+
batches: simple_abc.clone(),
1900+
projection: vec![
1901+
(col("b", &schema).unwrap(), "b".to_string()),
1902+
(col("a", &schema).unwrap(), "a".to_string()),
1903+
],
1904+
sort_expr: PhysicalSortExpr::new(
1905+
Arc::new(Column::new("a", 1)),
1906+
SortOptions::default(),
1907+
),
1908+
expected_plans: vec![
1909+
r#" - SortExec: TopK(fetch=2), expr=[a@1 ASC], preserve_partitioning=[false]
1910+
- ProjectionExec: expr=[b@1 as b, a@0 as a]
1911+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),
1912+
r#" - SortExec: TopK(fetch=2), expr=[a@1 ASC], preserve_partitioning=[false], filter=[a@1 IS NULL OR a@1 < 2]
1913+
- ProjectionExec: expr=[b@1 as b, a@0 as a]
1914+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"#.to_string()]
1915+
})
1916+
.await;
1917+
1918+
// Case 2: Pruning [a]
1919+
run_projection_dyn_filter_case(ProjectionDynFilterTestCase {
1920+
schema: Arc::clone(&schema),
1921+
batches: simple_abc.clone(),
1922+
projection: vec![(col("a", &schema).unwrap(), "a".to_string())],
1923+
sort_expr: PhysicalSortExpr::new(
1924+
Arc::new(Column::new("a", 0)),
1925+
SortOptions::default(),
1926+
),
1927+
expected_plans: vec![
1928+
r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false]
1929+
- ProjectionExec: expr=[a@0 as a]
1930+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),
1931+
r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false], filter=[a@0 IS NULL OR a@0 < 2]
1932+
- ProjectionExec: expr=[a@0 as a]
1933+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"#.to_string(),
1934+
],
1935+
})
1936+
.await;
1937+
1938+
// Case 3: Identity [a, b]
1939+
run_projection_dyn_filter_case(ProjectionDynFilterTestCase {
1940+
schema: Arc::clone(&schema),
1941+
batches: simple_abc.clone(),
1942+
projection: vec![
1943+
(col("a", &schema).unwrap(), "a".to_string()),
1944+
(col("b", &schema).unwrap(), "b".to_string()),
1945+
],
1946+
sort_expr: PhysicalSortExpr::new(
1947+
Arc::new(Column::new("a", 0)),
1948+
SortOptions::default(),
1949+
),
1950+
expected_plans: vec![
1951+
r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false]
1952+
- ProjectionExec: expr=[a@0 as a, b@1 as b]
1953+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),
1954+
r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false], filter=[a@0 IS NULL OR a@0 < 2]
1955+
- ProjectionExec: expr=[a@0 as a, b@1 as b]
1956+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"#.to_string(),
1957+
],
1958+
})
1959+
.await;
1960+
1961+
// Case 4: Expressions [a + 1, b]
1962+
run_projection_dyn_filter_case(ProjectionDynFilterTestCase {
1963+
schema: Arc::clone(&schema),
1964+
batches: simple_abc.clone(),
1965+
projection: vec![
1966+
(
1967+
Arc::new(BinaryExpr::new(
1968+
col("a", &schema).unwrap(),
1969+
Operator::Plus,
1970+
Arc::new(Literal::new(ScalarValue::Int32(Some(1)))),
1971+
)),
1972+
"a_plus_1".to_string(),
1973+
),
1974+
(col("b", &schema).unwrap(), "b".to_string()),
1975+
],
1976+
sort_expr: PhysicalSortExpr::new(
1977+
Arc::new(Column::new("a_plus_1", 0)),
1978+
SortOptions::default(),
1979+
),
1980+
expected_plans: vec![
1981+
r#" - SortExec: TopK(fetch=2), expr=[a_plus_1@0 ASC], preserve_partitioning=[false]
1982+
- ProjectionExec: expr=[a@0 + 1 as a_plus_1, b@1 as b]
1983+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),
1984+
r#" - SortExec: TopK(fetch=2), expr=[a_plus_1@0 ASC], preserve_partitioning=[false], filter=[a_plus_1@0 IS NULL OR a_plus_1@0 < 3]
1985+
- ProjectionExec: expr=[a@0 + 1 as a_plus_1, b@1 as b]
1986+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 + 1 IS NULL OR a@0 + 1 < 3 ]"#.to_string(),
1987+
],
1988+
})
1989+
.await;
1990+
1991+
// Case 5: [a as b, b as a] (swapped columns)
1992+
run_projection_dyn_filter_case(ProjectionDynFilterTestCase {
1993+
schema: Arc::clone(&schema),
1994+
batches: simple_abc.clone(),
1995+
projection: vec![
1996+
(col("a", &schema).unwrap(), "b".to_string()),
1997+
(col("b", &schema).unwrap(), "a".to_string()),
1998+
],
1999+
sort_expr: PhysicalSortExpr::new(
2000+
Arc::new(Column::new("b", 0)),
2001+
SortOptions::default(),
2002+
),
2003+
expected_plans: vec![
2004+
r#" - SortExec: TopK(fetch=2), expr=[b@0 ASC], preserve_partitioning=[false]
2005+
- ProjectionExec: expr=[a@0 as b, b@1 as a]
2006+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),
2007+
r#" - SortExec: TopK(fetch=2), expr=[b@0 ASC], preserve_partitioning=[false], filter=[b@0 IS NULL OR b@0 < 2]
2008+
- ProjectionExec: expr=[a@0 as b, b@1 as a]
2009+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"#.to_string(),
2010+
],
2011+
})
2012+
.await;
2013+
2014+
// Case 6: Confusing expr [a + 1 as a, b]
2015+
run_projection_dyn_filter_case(ProjectionDynFilterTestCase {
2016+
schema: Arc::clone(&schema),
2017+
batches: simple_abc.clone(),
2018+
projection: vec![
2019+
(
2020+
Arc::new(BinaryExpr::new(
2021+
col("a", &schema).unwrap(),
2022+
Operator::Plus,
2023+
Arc::new(Literal::new(ScalarValue::Int32(Some(1)))),
2024+
)),
2025+
"a".to_string(),
2026+
),
2027+
(col("b", &schema).unwrap(), "b".to_string()),
2028+
],
2029+
sort_expr: PhysicalSortExpr::new(
2030+
Arc::new(Column::new("a", 0)),
2031+
SortOptions::default(),
2032+
),
2033+
expected_plans: vec![
2034+
r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false]
2035+
- ProjectionExec: expr=[a@0 + 1 as a, b@1 as b]
2036+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),
2037+
r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false], filter=[a@0 IS NULL OR a@0 < 3]
2038+
- ProjectionExec: expr=[a@0 + 1 as a, b@1 as b]
2039+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 + 1 IS NULL OR a@0 + 1 < 3 ]"#.to_string(),
2040+
],
2041+
})
2042+
.await;
2043+
}
2044+
18162045
/// Returns a predicate that is a binary expression col = lit
18172046
fn col_lit_predicate(
18182047
column_name: &str,

‎datafusion/physical-plan/src/filter_pushdown.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,9 @@ use std::collections::HashSet;
3838
use std::sync::Arc;
3939

4040
use datafusion_common::Result;
41-
use datafusion_physical_expr::utils::{collect_columns, reassign_expr_columns};
41+
use datafusion_physical_expr::utils::{
42+
collect_columns, reassign_expr_columns,
43+
};
4244
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
4345
use itertools::Itertools;
4446

‎datafusion/physical-plan/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ pub mod streaming;
9292
pub mod tree_node;
9393
pub mod union;
9494
pub mod unnest;
95+
pub mod util;
9596
pub mod windows;
9697
pub mod work_table;
9798
pub mod udaf {

0 commit comments

Comments
 (0)