Skip to content

Commit e6fb0cd

Browse files
discord9de-bgunter
authored andcommitted
feat: support pushdown alias on dynamic filter with ProjectionExec (apache#19404)
## Which issue does this PR close? - Closes apache#17246 ## Rationale for this change For dynamic filter to work properly, table scan must get correct column even if it's passing through alias(by `ProjectionExec`) hence need to modify parent filter when `gather_filters_for_pushdown` ## What changes are included in this PR? as title, add support for handling simple alias in pushdown filter, which expand aliased column(in pushdown filter) to it's original expressions(or not pushdown if can't found aliased column in pushdown filter) so alias in projection is supported, also added unit tests. AI Content disclaim: the core logic is hand written and thoroughly understood, but unit test are largely generated with some human guidance ## Are these changes tested? Unit tests&slt are added, please comment if more tests are needed ## Are there any user-facing changes? Yes, dynamic filter will work properly with alias now, I'm not sure if that count as breaking change though? <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> <!-- If there are any breaking changes to public APIs, please add the `api change` label. --> --------- Signed-off-by: discord9 <[email protected]>
1 parent 3ec4a26 commit e6fb0cd

9 files changed

Lines changed: 1190 additions & 27 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/core/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,7 @@ env_logger = { workspace = true }
175175
glob = { workspace = true }
176176
insta = { workspace = true }
177177
paste = { workspace = true }
178+
pretty_assertions = "1.0"
178179
rand = { workspace = true, features = ["small_rng"] }
179180
rand_distr = "0.5"
180181
recursive = { workspace = true }

‎datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs‎

Lines changed: 229 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ use datafusion_physical_plan::{
5959
coalesce_partitions::CoalescePartitionsExec,
6060
collect,
6161
filter::{FilterExec, FilterExecBuilder},
62+
projection::ProjectionExec,
6263
repartition::RepartitionExec,
6364
sorts::sort::SortExec,
6465
};
@@ -1826,6 +1827,234 @@ fn schema() -> SchemaRef {
18261827
Arc::clone(&TEST_SCHEMA)
18271828
}
18281829

1830+
struct ProjectionDynFilterTestCase {
1831+
schema: SchemaRef,
1832+
batches: Vec<RecordBatch>,
1833+
projection: Vec<(Arc<dyn PhysicalExpr>, String)>,
1834+
sort_expr: PhysicalSortExpr,
1835+
expected_plans: Vec<String>,
1836+
}
1837+
1838+
async fn run_projection_dyn_filter_case(case: ProjectionDynFilterTestCase) {
1839+
let ProjectionDynFilterTestCase {
1840+
schema,
1841+
batches,
1842+
projection,
1843+
sort_expr,
1844+
expected_plans,
1845+
} = case;
1846+
1847+
let scan = TestScanBuilder::new(Arc::clone(&schema))
1848+
.with_support(true)
1849+
.with_batches(batches)
1850+
.build();
1851+
1852+
let projection_exec = Arc::new(ProjectionExec::try_new(projection, scan).unwrap());
1853+
1854+
let sort = Arc::new(
1855+
SortExec::new(LexOrdering::new(vec![sort_expr]).unwrap(), projection_exec)
1856+
.with_fetch(Some(2)),
1857+
) as Arc<dyn ExecutionPlan>;
1858+
1859+
let mut config = ConfigOptions::default();
1860+
config.execution.parquet.pushdown_filters = true;
1861+
config.optimizer.enable_dynamic_filter_pushdown = true;
1862+
1863+
let optimized_plan = FilterPushdown::new_post_optimization()
1864+
.optimize(Arc::clone(&sort), &config)
1865+
.unwrap();
1866+
1867+
pretty_assertions::assert_eq!(
1868+
format_plan_for_test(&optimized_plan).trim(),
1869+
expected_plans[0].trim()
1870+
);
1871+
1872+
let config = SessionConfig::new().with_batch_size(2);
1873+
let session_ctx = SessionContext::new_with_config(config);
1874+
session_ctx.register_object_store(
1875+
ObjectStoreUrl::parse("test://").unwrap().as_ref(),
1876+
Arc::new(InMemory::new()),
1877+
);
1878+
let state = session_ctx.state();
1879+
let task_ctx = state.task_ctx();
1880+
let mut stream = optimized_plan.execute(0, Arc::clone(&task_ctx)).unwrap();
1881+
for (idx, expected_plan) in expected_plans.iter().enumerate().skip(1) {
1882+
stream.next().await.unwrap().unwrap();
1883+
let formatted_plan = format_plan_for_test(&optimized_plan);
1884+
pretty_assertions::assert_eq!(
1885+
formatted_plan.trim(),
1886+
expected_plan.trim(),
1887+
"Mismatch at iteration {}",
1888+
idx
1889+
);
1890+
}
1891+
}
1892+
1893+
#[tokio::test]
1894+
async fn test_topk_with_projection_transformation_on_dyn_filter() {
1895+
let schema = Arc::new(Schema::new(vec![
1896+
Field::new("a", DataType::Int32, false),
1897+
Field::new("b", DataType::Utf8, false),
1898+
Field::new("c", DataType::Float64, false),
1899+
]));
1900+
let simple_abc = vec![
1901+
record_batch!(
1902+
("a", Int32, [1, 2, 3]),
1903+
("b", Utf8, ["x", "y", "z"]),
1904+
("c", Float64, [1.0, 2.0, 3.0])
1905+
)
1906+
.unwrap(),
1907+
];
1908+
1909+
// Case 1: Reordering [b, a]
1910+
run_projection_dyn_filter_case(ProjectionDynFilterTestCase {
1911+
schema: Arc::clone(&schema),
1912+
batches: simple_abc.clone(),
1913+
projection: vec![
1914+
(col("b", &schema).unwrap(), "b".to_string()),
1915+
(col("a", &schema).unwrap(), "a".to_string()),
1916+
],
1917+
sort_expr: PhysicalSortExpr::new(
1918+
Arc::new(Column::new("a", 1)),
1919+
SortOptions::default(),
1920+
),
1921+
expected_plans: vec![
1922+
r#" - SortExec: TopK(fetch=2), expr=[a@1 ASC], preserve_partitioning=[false]
1923+
- ProjectionExec: expr=[b@1 as b, a@0 as a]
1924+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),
1925+
r#" - SortExec: TopK(fetch=2), expr=[a@1 ASC], preserve_partitioning=[false], filter=[a@1 IS NULL OR a@1 < 2]
1926+
- ProjectionExec: expr=[b@1 as b, a@0 as a]
1927+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"#.to_string()]
1928+
})
1929+
.await;
1930+
1931+
// Case 2: Pruning [a]
1932+
run_projection_dyn_filter_case(ProjectionDynFilterTestCase {
1933+
schema: Arc::clone(&schema),
1934+
batches: simple_abc.clone(),
1935+
projection: vec![(col("a", &schema).unwrap(), "a".to_string())],
1936+
sort_expr: PhysicalSortExpr::new(
1937+
Arc::new(Column::new("a", 0)),
1938+
SortOptions::default(),
1939+
),
1940+
expected_plans: vec![
1941+
r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false]
1942+
- ProjectionExec: expr=[a@0 as a]
1943+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),
1944+
r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false], filter=[a@0 IS NULL OR a@0 < 2]
1945+
- ProjectionExec: expr=[a@0 as a]
1946+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"#.to_string(),
1947+
],
1948+
})
1949+
.await;
1950+
1951+
// Case 3: Identity [a, b]
1952+
run_projection_dyn_filter_case(ProjectionDynFilterTestCase {
1953+
schema: Arc::clone(&schema),
1954+
batches: simple_abc.clone(),
1955+
projection: vec![
1956+
(col("a", &schema).unwrap(), "a".to_string()),
1957+
(col("b", &schema).unwrap(), "b".to_string()),
1958+
],
1959+
sort_expr: PhysicalSortExpr::new(
1960+
Arc::new(Column::new("a", 0)),
1961+
SortOptions::default(),
1962+
),
1963+
expected_plans: vec![
1964+
r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false]
1965+
- ProjectionExec: expr=[a@0 as a, b@1 as b]
1966+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),
1967+
r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false], filter=[a@0 IS NULL OR a@0 < 2]
1968+
- ProjectionExec: expr=[a@0 as a, b@1 as b]
1969+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"#.to_string(),
1970+
],
1971+
})
1972+
.await;
1973+
1974+
// Case 4: Expressions [a + 1, b]
1975+
run_projection_dyn_filter_case(ProjectionDynFilterTestCase {
1976+
schema: Arc::clone(&schema),
1977+
batches: simple_abc.clone(),
1978+
projection: vec![
1979+
(
1980+
Arc::new(BinaryExpr::new(
1981+
col("a", &schema).unwrap(),
1982+
Operator::Plus,
1983+
Arc::new(Literal::new(ScalarValue::Int32(Some(1)))),
1984+
)),
1985+
"a_plus_1".to_string(),
1986+
),
1987+
(col("b", &schema).unwrap(), "b".to_string()),
1988+
],
1989+
sort_expr: PhysicalSortExpr::new(
1990+
Arc::new(Column::new("a_plus_1", 0)),
1991+
SortOptions::default(),
1992+
),
1993+
expected_plans: vec![
1994+
r#" - SortExec: TopK(fetch=2), expr=[a_plus_1@0 ASC], preserve_partitioning=[false]
1995+
- ProjectionExec: expr=[a@0 + 1 as a_plus_1, b@1 as b]
1996+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),
1997+
r#" - SortExec: TopK(fetch=2), expr=[a_plus_1@0 ASC], preserve_partitioning=[false], filter=[a_plus_1@0 IS NULL OR a_plus_1@0 < 3]
1998+
- ProjectionExec: expr=[a@0 + 1 as a_plus_1, b@1 as b]
1999+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 + 1 IS NULL OR a@0 + 1 < 3 ]"#.to_string(),
2000+
],
2001+
})
2002+
.await;
2003+
2004+
// Case 5: [a as b, b as a] (swapped columns)
2005+
run_projection_dyn_filter_case(ProjectionDynFilterTestCase {
2006+
schema: Arc::clone(&schema),
2007+
batches: simple_abc.clone(),
2008+
projection: vec![
2009+
(col("a", &schema).unwrap(), "b".to_string()),
2010+
(col("b", &schema).unwrap(), "a".to_string()),
2011+
],
2012+
sort_expr: PhysicalSortExpr::new(
2013+
Arc::new(Column::new("b", 0)),
2014+
SortOptions::default(),
2015+
),
2016+
expected_plans: vec![
2017+
r#" - SortExec: TopK(fetch=2), expr=[b@0 ASC], preserve_partitioning=[false]
2018+
- ProjectionExec: expr=[a@0 as b, b@1 as a]
2019+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),
2020+
r#" - SortExec: TopK(fetch=2), expr=[b@0 ASC], preserve_partitioning=[false], filter=[b@0 IS NULL OR b@0 < 2]
2021+
- ProjectionExec: expr=[a@0 as b, b@1 as a]
2022+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"#.to_string(),
2023+
],
2024+
})
2025+
.await;
2026+
2027+
// Case 6: Confusing expr [a + 1 as a, b]
2028+
run_projection_dyn_filter_case(ProjectionDynFilterTestCase {
2029+
schema: Arc::clone(&schema),
2030+
batches: simple_abc.clone(),
2031+
projection: vec![
2032+
(
2033+
Arc::new(BinaryExpr::new(
2034+
col("a", &schema).unwrap(),
2035+
Operator::Plus,
2036+
Arc::new(Literal::new(ScalarValue::Int32(Some(1)))),
2037+
)),
2038+
"a".to_string(),
2039+
),
2040+
(col("b", &schema).unwrap(), "b".to_string()),
2041+
],
2042+
sort_expr: PhysicalSortExpr::new(
2043+
Arc::new(Column::new("a", 0)),
2044+
SortOptions::default(),
2045+
),
2046+
expected_plans: vec![
2047+
r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false]
2048+
- ProjectionExec: expr=[a@0 + 1 as a, b@1 as b]
2049+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),
2050+
r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false], filter=[a@0 IS NULL OR a@0 < 3]
2051+
- ProjectionExec: expr=[a@0 + 1 as a, b@1 as b]
2052+
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 + 1 IS NULL OR a@0 + 1 < 3 ]"#.to_string(),
2053+
],
2054+
})
2055+
.await;
2056+
}
2057+
18292058
/// Returns a predicate that is a binary expression col = lit
18302059
fn col_lit_predicate(
18312060
column_name: &str,

0 commit comments

Comments
 (0)