Skip to content

Commit dd4dcbc

Browse files
authored
fix: Handle input_file_name() pushdown into ParquetSource with filter pushdown enabled (apache#23638)
## Which issue does this PR close? - Closes apache#23531. ## Rationale for this change `input_file_name()` is `FileSource` dependent like `file_row_index()`, and therefore shouldn't be pushed down into a filter. ## What changes are included in this PR? `PushdownChecker` now handles both UDFs consistently. If we keep adding this sort of metadata functions, we might want a better API to detect them, but for now I think this is a reasonable approach that isn't very invasive. ## Are these changes tested? Additional SLT test that verifies that both function behave correctly with pushdown either enabled or disabled. ## Are there any user-facing changes? None --------- Signed-off-by: Adam Gutglick <[email protected]>
1 parent 50ae076 commit dd4dcbc

3 files changed

Lines changed: 34 additions & 4 deletions

File tree

‎datafusion/datasource-parquet/src/projection_read_plan.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ use std::collections::{BTreeMap, BTreeSet};
3030
use std::sync::Arc;
3131

3232
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
33+
use datafusion_functions::core::input_file_name::InputFileNameFunc;
3334
use parquet::arrow::ProjectionMask;
3435
use parquet::schema::types::SchemaDescriptor;
3536

@@ -313,8 +314,10 @@ impl TreeNodeVisitor<'_> for PushdownChecker<'_> {
313314
return Ok(recursion);
314315
}
315316

316-
if ScalarFunctionExpr::try_downcast_func::<FileRowIndexFunc>(node.as_ref())
317+
if ScalarFunctionExpr::try_downcast_func::<InputFileNameFunc>(node.as_ref())
317318
.is_some()
319+
|| ScalarFunctionExpr::try_downcast_func::<FileRowIndexFunc>(node.as_ref())
320+
.is_some()
318321
{
319322
self.has_unpushable_udfs = true;
320323
return Ok(TreeNodeRecursion::Jump);

‎datafusion/sqllogictest/test_files/input_file_name.slt‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -121,8 +121,7 @@ physical_plan
121121
01)SortPreservingMergeExec: [column1@0 ASC NULLS LAST]
122122
02)--SortExec: expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[true]
123123
03)----FilterExec: __datafusion_extracted_1@0 LIKE %first.parquet, projection=[column1@1]
124-
04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2
125-
05)--------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/first.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/second.parquet]]}, projection=[input_file_name() as __datafusion_extracted_1, column1], file_type=parquet, predicate=input_file_name() LIKE %first.parquet
124+
04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/first.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/second.parquet]]}, projection=[input_file_name() as __datafusion_extracted_1, column1], file_type=parquet
126125

127126
statement ok
128-
DROP TABLE pq_table;
127+
DROP TABLE pq_table;

‎datafusion/sqllogictest/test_files/parquet_metadata_functions.slt‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,5 +52,33 @@ logical_plan
5252
02)--TableScan: test_table projection=[column1]
5353
physical_plan DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/first.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/second.parquet]]}, projection=[input_file_name() as input_file_name(), CAST(__datafusion_file_row_index@1 AS Int64) as file_row_index(), column1], file_type=parquet
5454

55+
56+
# Make sure it also behaves consistently regardless of filter pushdown
57+
58+
statement ok
59+
SET datafusion.execution.parquet.pushdown_filters = false;
60+
61+
query TII rowsort
62+
SELECT input_file_name(), file_row_index(), column1
63+
FROM test_table
64+
WHERE file_row_index() = 2 AND input_file_name() LIKE '%parquet';
65+
----
66+
WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/first.parquet 2 30
67+
WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/second.parquet 2 60
68+
69+
statement ok
70+
SET datafusion.execution.parquet.pushdown_filters = true;
71+
72+
query TII rowsort
73+
SELECT input_file_name(), file_row_index(), column1
74+
FROM test_table
75+
WHERE file_row_index() = 2 AND input_file_name() LIKE '%parquet';
76+
----
77+
WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/first.parquet 2 30
78+
WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/second.parquet 2 60
79+
80+
statement ok
81+
RESET datafusion.execution.parquet.pushdown_filters;
82+
5583
statement ok
5684
DROP TABLE test_table;

0 commit comments

Comments
 (0)