Skip to content

Commit 60c407c

Browse files
AdamGSalamb
andauthored
feat: Add new input_file_name UDF for file-backed scans (#22978)
## Which issue does this PR close? - Closes #6051. - Closes #20071 - Part of #20135. ## Rationale for this change Adds useful metadata functions to DataFusion. This PR builds on @ethan-tyler's #20071. ## What changes are included in this PR? 1. A new `input_file_name()` UDF, which reports a UTF8 return_type. Like `file_row_index()`, it errors when evaluated out of context. 2. Add rewrites in the `FileOpener` (Both in `ParquetOpener` and `ProjectionOpener`), making it a literal with the file's path. 3. New public rewrite helper `rewrite_input_file_name_in_projection` in `datafusion-physical-expr-adapter`. ## Are these changes tested? 1. Unit tests in all rewrite-sites and for the core rewrite logic 2. New SLT tests working on CSV 3. New SLT testing metadata functions specifically on Parquet which has its own opener. These include `file_row_index()`. ## Are there any user-facing changes? - The new UDF - New public function in `datafusion-physical-expr-adapter` - `rewrite_input_file_name_in_projection`. AI was used in this PR, mostly when helping to come up with test cases. --------- Signed-off-by: Adam Gutglick <[email protected]> Co-authored-by: Andrew Lamb <[email protected]>
1 parent a4c4b4d commit 60c407c

10 files changed

Lines changed: 540 additions & 15 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/datasource-parquet/src/opener/mod.rs‎

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ use arrow::datatypes::DataType;
4040
use datafusion_datasource::morsel::{Morsel, MorselPlan, MorselPlanner, Morselizer};
4141
use datafusion_physical_expr::projection::ProjectionExprs;
4242
use datafusion_physical_expr_adapter::replace_columns_with_literals;
43+
use datafusion_physical_expr_adapter::schema_rewriter::rewrite_input_file_name_in_projection;
4344
use std::collections::{HashMap, VecDeque};
4445
use std::fmt;
4546
use std::future::Future;
@@ -795,6 +796,9 @@ impl ParquetMorselizer {
795796
.transpose()?;
796797
}
797798

799+
// Replace any `input_file_name()` UDFs in the projection with a literal for this file.
800+
projection = rewrite_input_file_name_in_projection(projection, &file_name)?;
801+
798802
let predicate_creation_errors = MetricBuilder::new(&self.metrics)
799803
.with_category(MetricCategory::Rows)
800804
.global_counter("num_predicate_creation_errors");
@@ -3280,8 +3284,12 @@ mod test {
32803284
/// (e.g. `row_number`) plumbed through `TableSchema`/`ParquetOpener`.
32813285
mod virtual_columns {
32823286
use super::*;
3283-
use arrow::array::{Array, Int64Array};
3287+
use arrow::array::{Array, Int64Array, StringArray};
32843288
use arrow::datatypes::FieldRef;
3289+
use datafusion_common::config::ConfigOptions;
3290+
use datafusion_expr::ScalarUDF;
3291+
use datafusion_functions::core::input_file_name::InputFileNameFunc;
3292+
use datafusion_physical_expr::{ScalarFunctionExpr, projection::ProjectionExpr};
32853293
use parquet::arrow::RowNumber;
32863294

32873295
/// Build a parquet `row_number` virtual column field. Spark's
@@ -3295,6 +3303,16 @@ mod test {
32953303
)
32963304
}
32973305

3306+
fn input_file_name_expr() -> Arc<dyn PhysicalExpr> {
3307+
Arc::new(ScalarFunctionExpr::new(
3308+
"input_file_name",
3309+
Arc::new(ScalarUDF::from(InputFileNameFunc::new())),
3310+
vec![],
3311+
Arc::new(Field::new("input_file_name", DataType::Utf8, true)),
3312+
Arc::new(ConfigOptions::default()),
3313+
))
3314+
}
3315+
32983316
/// Collect every `Int64` value from the given column in every batch
32993317
/// of a stream. Used to verify the `row_number` column end to end.
33003318
async fn collect_int64_values(
@@ -3414,6 +3432,44 @@ mod test {
34143432
assert_eq!(row_numbers, vec![0, 1, 2, 3]);
34153433
}
34163434

3435+
#[tokio::test]
3436+
async fn test_input_file_name_projection() {
3437+
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
3438+
let path = "dir/input_file_name.parquet";
3439+
let (file_schema, data_size) = write_grouped_file(&store, path, 1, 3).await;
3440+
3441+
let projection = ProjectionExprs::new([
3442+
ProjectionExpr::new(Arc::new(Column::new("value", 0)), "value"),
3443+
ProjectionExpr::new(input_file_name_expr(), "file_name"),
3444+
]);
3445+
3446+
let morselizer = ParquetMorselizerBuilder::new()
3447+
.with_store(Arc::clone(&store))
3448+
.with_schema(file_schema)
3449+
.with_projection(projection)
3450+
.build();
3451+
3452+
let file =
3453+
PartitionedFile::new(path.to_string(), u64::try_from(data_size).unwrap());
3454+
let mut stream = open_file(&morselizer, file).await.unwrap();
3455+
let batch = stream.next().await.unwrap().unwrap();
3456+
assert!(stream.next().await.is_none());
3457+
3458+
assert_eq!(batch.num_columns(), 2);
3459+
assert_eq!(batch.schema().field(0).name(), "value");
3460+
assert_eq!(batch.schema().field(1).name(), "file_name");
3461+
3462+
let file_names = batch
3463+
.column(1)
3464+
.as_any()
3465+
.downcast_ref::<StringArray>()
3466+
.expect("file_name column should be Utf8");
3467+
assert_eq!(file_names.len(), 3);
3468+
for i in 0..file_names.len() {
3469+
assert_eq!(file_names.value(i), path);
3470+
}
3471+
}
3472+
34173473
#[tokio::test]
34183474
async fn test_row_index_multi_row_group() {
34193475
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;

‎datafusion/datasource/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ zstd = { workspace = true, optional = true }
7474

7575
[dev-dependencies]
7676
criterion = { workspace = true }
77+
datafusion-functions = { workspace = true }
7778
insta = { workspace = true }
7879
tempfile = { workspace = true }
7980

‎datafusion/datasource/src/projection.rs‎

Lines changed: 90 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ use datafusion_physical_expr::{
2626
expressions::{Column, Literal},
2727
projection::{ProjectionExpr, ProjectionExprs},
2828
};
29+
use datafusion_physical_expr_adapter::schema_rewriter::rewrite_input_file_name_in_projection;
2930
use futures::{FutureExt, StreamExt};
3031
use itertools::Itertools;
3132

@@ -69,6 +70,7 @@ impl ProjectionOpener {
6970
impl FileOpener for ProjectionOpener {
7071
fn open(&self, partitioned_file: PartitionedFile) -> Result<FileOpenFuture> {
7172
let partition_values = partitioned_file.partition_values.clone();
73+
7274
// Modify any references to partition columns in the projection expressions
7375
// and substitute them with literal values from PartitionedFile.partition_values
7476
let projection = if self.partition_columns.is_empty() {
@@ -80,6 +82,11 @@ impl FileOpener for ProjectionOpener {
8082
partition_values,
8183
)
8284
};
85+
// Replace `input_file_name()` with a per-file literal if present.
86+
let projection = rewrite_input_file_name_in_projection(
87+
projection,
88+
partitioned_file.object_meta.location.as_ref(),
89+
)?;
8390
let projector = projection.make_projector(&self.input_schema)?;
8491

8592
let inner = self.inner.open(partitioned_file)?;
@@ -287,15 +294,31 @@ impl SplitProjection {
287294
mod test {
288295
use std::sync::Arc;
289296

290-
use arrow::array::AsArray;
291-
use arrow::datatypes::{DataType, SchemaRef};
292-
use datafusion_common::{DFSchema, ScalarValue, record_batch};
293-
use datafusion_expr::{Expr, col, execution_props::ExecutionProps};
294-
use datafusion_physical_expr::{create_physical_exprs, projection::ProjectionExpr};
297+
use arrow::array::{AsArray, RecordBatch};
298+
use arrow::datatypes::{DataType, Field, SchemaRef};
299+
use datafusion_common::{DFSchema, ScalarValue, config::ConfigOptions, record_batch};
300+
use datafusion_expr::{Expr, ScalarUDF, col, execution_props::ExecutionProps};
301+
use datafusion_functions::core::input_file_name::InputFileNameFunc;
302+
use datafusion_physical_expr::{
303+
ScalarFunctionExpr, create_physical_exprs, projection::ProjectionExpr,
304+
};
305+
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
306+
use futures::{FutureExt, StreamExt};
295307
use itertools::Itertools;
296308

297309
use super::*;
298310

311+
struct StaticBatchOpener {
312+
batch: RecordBatch,
313+
}
314+
315+
impl FileOpener for StaticBatchOpener {
316+
fn open(&self, _partitioned_file: PartitionedFile) -> Result<FileOpenFuture> {
317+
let batch = self.batch.clone();
318+
Ok(async move { Ok(futures::stream::iter([Ok(batch)]).boxed()) }.boxed())
319+
}
320+
}
321+
299322
fn create_projection_exprs<'a>(
300323
exprs: impl IntoIterator<Item = &'a Expr>,
301324
schema: &SchemaRef,
@@ -311,6 +334,68 @@ mod test {
311334
ProjectionExprs::from(projection_exprs)
312335
}
313336

337+
fn input_file_name_expr() -> Arc<dyn PhysicalExpr> {
338+
Arc::new(ScalarFunctionExpr::new(
339+
"input_file_name",
340+
Arc::new(ScalarUDF::from(InputFileNameFunc::new())),
341+
vec![],
342+
Arc::new(Field::new("input_file_name", DataType::Utf8, true)),
343+
Arc::new(ConfigOptions::default()),
344+
))
345+
}
346+
347+
#[tokio::test]
348+
async fn test_projection_opener_rewrites_input_file_name_with_partitions() {
349+
let file_schema = Schema::new(vec![Field::new("value", DataType::Int32, false)]);
350+
let projection = ProjectionExprs::new([
351+
ProjectionExpr::new(Arc::new(Column::new("value", 0)), "value"),
352+
ProjectionExpr::new(Arc::new(Column::new("part", 1)), "part"),
353+
ProjectionExpr::new(input_file_name_expr(), "file_name"),
354+
]);
355+
let split = SplitProjection::new(&file_schema, &projection);
356+
let input_batch =
357+
record_batch!(("value", Int32, vec![10, 20])).expect("input batch");
358+
359+
let opener = ProjectionOpener::try_new(
360+
split,
361+
Arc::new(StaticBatchOpener { batch: input_batch }),
362+
&file_schema,
363+
)
364+
.expect("projection opener");
365+
366+
let mut file = PartitionedFile::new("part=west/data.csv", 100);
367+
file.partition_values = vec![ScalarValue::from("west")];
368+
let mut stream = opener
369+
.open(file)
370+
.expect("open projection")
371+
.await
372+
.expect("inner stream");
373+
let batch = stream
374+
.next()
375+
.await
376+
.expect("one projected batch")
377+
.expect("projected batch");
378+
assert!(stream.next().await.is_none());
379+
380+
assert_eq!(batch.schema().field(0).name(), "value");
381+
assert_eq!(batch.schema().field(1).name(), "part");
382+
assert_eq!(batch.schema().field(2).name(), "file_name");
383+
384+
let values = batch
385+
.column(0)
386+
.as_primitive::<arrow::datatypes::Int32Type>();
387+
assert_eq!(values.value(0), 10);
388+
assert_eq!(values.value(1), 20);
389+
390+
let parts = batch.column(1).as_string::<i32>();
391+
assert_eq!(parts.value(0), "west");
392+
assert_eq!(parts.value(1), "west");
393+
394+
let file_names = batch.column(2).as_string::<i32>();
395+
assert_eq!(file_names.value(0), "part=west/data.csv");
396+
assert_eq!(file_names.value(1), "part=west/data.csv");
397+
}
398+
314399
#[test]
315400
fn test_split_projection_with_partition_columns() {
316401
use arrow::array::AsArray;
Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! [`InputFileNameFunc`]: Implementation of the `input_file_name` function.
19+
20+
use arrow::datatypes::DataType;
21+
use datafusion_common::{exec_err, utils::take_function_args};
22+
use datafusion_doc::Documentation;
23+
use datafusion_expr::{
24+
ColumnarValue, ExpressionPlacement, ScalarFunctionArgs, ScalarUDFImpl, Signature,
25+
Volatility,
26+
};
27+
use datafusion_macros::user_doc;
28+
29+
#[user_doc(
30+
doc_section(label = "Other Functions"),
31+
description = r#"Returns the path of the input file that produced the current row.
32+
33+
Note: file paths/URIs may be sensitive metadata depending on your environment.
34+
35+
This function is intended to be rewritten at file-scan time (when the file is
36+
known). If the input file is not known (for example, if this function is
37+
evaluated outside a file scan, or was not pushed down into one), direct evaluation returns an error.
38+
"#,
39+
syntax_example = "input_file_name()",
40+
sql_example = r#"```sql
41+
SELECT input_file_name() FROM t;
42+
```"#
43+
)]
44+
#[derive(Debug, PartialEq, Eq, Hash)]
45+
pub struct InputFileNameFunc {
46+
signature: Signature,
47+
}
48+
49+
impl Default for InputFileNameFunc {
50+
fn default() -> Self {
51+
Self::new()
52+
}
53+
}
54+
55+
impl InputFileNameFunc {
56+
pub fn new() -> Self {
57+
Self {
58+
signature: Signature::nullary(Volatility::Volatile),
59+
}
60+
}
61+
}
62+
63+
impl ScalarUDFImpl for InputFileNameFunc {
64+
fn name(&self) -> &str {
65+
"input_file_name"
66+
}
67+
68+
fn signature(&self) -> &Signature {
69+
&self.signature
70+
}
71+
72+
fn return_type(&self, arg_types: &[DataType]) -> datafusion_common::Result<DataType> {
73+
let [] = take_function_args(self.name(), arg_types)?;
74+
Ok(DataType::Utf8)
75+
}
76+
77+
fn invoke_with_args(
78+
&self,
79+
args: ScalarFunctionArgs,
80+
) -> datafusion_common::Result<ColumnarValue> {
81+
let [] = take_function_args(self.name(), args.args)?;
82+
83+
exec_err!(
84+
"input_file_name() is source dependent and cannot be evaluated directly"
85+
)
86+
}
87+
88+
fn placement(&self, _args: &[ExpressionPlacement]) -> ExpressionPlacement {
89+
ExpressionPlacement::MoveTowardsLeafNodes
90+
}
91+
92+
fn documentation(&self) -> Option<&Documentation> {
93+
self.doc()
94+
}
95+
}

‎datafusion/functions/src/core/mod.rs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ pub mod file_row_index;
3232
pub mod getfield;
3333
pub mod greatest;
3434
mod greatest_least_utils;
35+
pub mod input_file_name;
3536
pub mod least;
3637
pub mod named_struct;
3738
pub mod nullif;
@@ -69,6 +70,7 @@ make_udf_function!(arrow_metadata::ArrowMetadataFunc, arrow_metadata);
6970
make_udf_function!(with_metadata::WithMetadataFunc, with_metadata);
7071
make_udf_function!(arrow_field::ArrowFieldFunc, arrow_field);
7172
make_udf_function!(file_row_index::FileRowIndexFunc, file_row_index);
73+
make_udf_function!(input_file_name::InputFileNameFunc, input_file_name);
7274

7375
pub mod expr_fn {
7476
use datafusion_expr::{Expr, Literal};
@@ -117,7 +119,12 @@ pub mod expr_fn {
117119
arrow_metadata,
118120
"Returns the metadata of the input expression",
119121
args,
120-
),(
122+
),
123+
(
124+
input_file_name,
125+
"Returns the path of the input file that produced the current row",
126+
),
127+
(
121128
with_metadata,
122129
"Attaches Arrow field metadata (key/value pairs) to the input expression",
123130
args,
@@ -200,6 +207,7 @@ pub fn functions() -> Vec<Arc<ScalarUDF>> {
200207
union_extract(),
201208
union_tag(),
202209
version(),
210+
input_file_name(),
203211
r#struct(),
204212
file_row_index(),
205213
]

0 commit comments

Comments
 (0)