Skip to content

Commit 5fabdb1

Browse files
authored
fix: keep field metadata when the Parquet scan replaces columns with literals (apache#26856)
## Which issue does this PR close? - Closes apache#26855. ## Rationale for this change Expressions pushed into a Parquet scan, such as `arrow_metadata(col)` or extension-aware UDFs, can see a column without its field metadata: `SELECT arrow_metadata(u) FROM t` returns `{}` although the table schema has `ARROW:extension:name` on `u`. See apache#26855 for a reproducer. The Parquet opener replaces column references with literals for partition columns and for columns that the file statistics prove constant (`min == max`). The literals had no field metadata, so expressions evaluated on them lost the extension type. This happens for example for every column of a one-row file. ## What changes are included in this PR? - New function `replace_columns_with_literals_keeping_metadata` in `datafusion-physical-expr-adapter`, next to `replace_columns_with_literals`. It takes the metadata of each literal from the field with the same name in a given schema. `DefaultPhysicalExprAdapter` already creates its literals for missing columns the same way. - The Parquet opener uses it with the table schema, for the projection and for the predicate. `replace_columns_with_literals` is unchanged. ## What is the testing strategy for this PR? - `pushed_down_projection_sees_field_metadata` in `datafusion/core/tests/parquet/schema.rs`: `arrow_metadata(id, 'ARROW:extension:name')` over a one-row Parquet file. - A new case in `cast_extension_type_metadata.slt`: a `UUID` column written with `COPY ... TO` and read back. It sets `skip_metadata = false` because until apache#26854 is merged, schema inference drops extension types by default (apache#26851). - A unit test for `replace_columns_with_literals_keeping_metadata`. Both integration tests fail without the change in the opener. ## Are there any user-facing changes? Expressions evaluated in a Parquet scan now see the field metadata of the table schema for partition columns and for columns that the statistics prove constant. There is one new public function, `replace_columns_with_literals_keeping_metadata`, and no breaking API changes.
1 parent 5787b60 commit 5fabdb1

5 files changed

Lines changed: 153 additions & 9 deletions

File tree

‎datafusion/core/tests/parquet/schema.rs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,40 @@ async fn schema_merge_can_preserve_metadata() {
176176
assert_metadata(&actual, &expected_metadata);
177177
}
178178

179+
/// Expressions pushed into the parquet scan see the field metadata of the table schema
180+
#[tokio::test]
181+
async fn pushed_down_projection_sees_field_metadata() {
182+
let tmp_dir = TempDir::new().unwrap();
183+
let table_dir = tmp_dir.path().join("parquet_test");
184+
185+
let f1 = Field::new("id", DataType::Int32, true)
186+
.with_metadata(make_meta("ARROW:extension:name", "example.ext"));
187+
let f2 = Field::new("name", DataType::Utf8, true);
188+
write_files(table_dir.as_path(), vec![Schema::new(vec![f1, f2])]);
189+
let table_path = table_dir.to_str().unwrap().to_string();
190+
191+
let ctx = SessionContext::new();
192+
let options = ParquetReadOptions::default().skip_metadata(false);
193+
ctx.register_parquet("t", &table_path, options)
194+
.await
195+
.unwrap();
196+
197+
let actual = ctx
198+
.sql("SELECT arrow_metadata(id, 'ARROW:extension:name') AS m FROM t")
199+
.await
200+
.unwrap()
201+
.collect()
202+
.await
203+
.unwrap();
204+
assert_snapshot!(batches_to_sort_string(&actual), @r"
205+
+-------------+
206+
| m |
207+
+-------------+
208+
| example.ext |
209+
+-------------+
210+
");
211+
}
212+
179213
/// A column that only some files carry must be inferred as nullable, because
180214
/// reading the files without it yields nulls, even when every file that has
181215
/// the column declares it required.

‎datafusion/datasource-parquet/src/opener/mod.rs‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,10 @@ use arrow::array::RecordBatch;
2929
use arrow::datatypes::DataType;
3030
use datafusion_datasource::morsel::{Morsel, MorselPlan, MorselPlanner, Morselizer};
3131
use datafusion_physical_expr::projection::ProjectionExprs;
32-
use datafusion_physical_expr_adapter::replace_columns_with_literals;
3332
use datafusion_physical_expr_adapter::rewrite::rewrite_input_file_name_in_projection;
33+
use datafusion_physical_expr_adapter::{
34+
replace_columns_with_literals, replace_columns_with_literals_keeping_metadata,
35+
};
3436
use std::collections::{HashMap, VecDeque};
3537
use std::fmt;
3638
use std::future::Future;
@@ -922,11 +924,23 @@ impl ParquetMorselizer {
922924
let mut projection = self.projection.clone();
923925
let mut predicate = self.predicate.clone();
924926
if !literal_columns.is_empty() {
927+
// Keep the field metadata of the replaced columns, such as extension types
928+
let table_schema = self.table_schema.table_schema();
925929
projection = projection.try_map_exprs(|expr| {
926-
replace_columns_with_literals(Arc::clone(&expr), &literal_columns)
930+
replace_columns_with_literals_keeping_metadata(
931+
Arc::clone(&expr),
932+
&literal_columns,
933+
table_schema,
934+
)
927935
})?;
928936
predicate = predicate
929-
.map(|p| replace_columns_with_literals(p, &literal_columns))
937+
.map(|p| {
938+
replace_columns_with_literals_keeping_metadata(
939+
p,
940+
&literal_columns,
941+
table_schema,
942+
)
943+
})
930944
.transpose()?;
931945
}
932946

‎datafusion/physical-expr-adapter/src/lib.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,5 @@ pub mod schema_rewriter;
1717
pub use schema_rewriter::{
1818
BatchAdapter, BatchAdapterFactory, DefaultPhysicalExprAdapter,
1919
DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,
20-
replace_columns_with_literals,
20+
replace_columns_with_literals, replace_columns_with_literals_keeping_metadata,
2121
};

‎datafusion/physical-expr-adapter/src/schema_rewriter.rs‎

Lines changed: 73 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ use std::hash::Hash;
1212
use std::sync::Arc;
1313

1414
use arrow::array::RecordBatch;
15-
use arrow::datatypes::{DataType, FieldRef, Fields, SchemaRef};
15+
use arrow::datatypes::{DataType, FieldRef, Fields, Schema, SchemaRef};
1616
use datafusion_common::{
1717
DataFusionError, Result, ScalarValue, exec_err,
1818
metadata::FieldMetadata,
@@ -25,7 +25,7 @@ use datafusion_physical_expr::expressions::Literal;
2525
use datafusion_physical_expr::projection::{ProjectionExprs, Projector};
2626
use datafusion_physical_expr::{
2727
ScalarFunctionExpr,
28-
expressions::{self, CastExpr, Column},
28+
expressions::{CastExpr, Column},
2929
};
3030
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
3131
use itertools::Itertools;
@@ -52,6 +52,38 @@ pub fn replace_columns_with_literals<K, V>(
5252
expr: Arc<dyn PhysicalExpr>,
5353
replacements: &HashMap<K, V>,
5454
) -> Result<Arc<dyn PhysicalExpr>>
55+
where
56+
K: Borrow<str> + Eq + Hash,
57+
V: Borrow<ScalarValue>,
58+
{
59+
replace_columns_with_literals_impl(expr, replacements, None)
60+
}
61+
62+
/// Replace column references in the given physical expression with literal values,
63+
/// like [`replace_columns_with_literals`], keeping the field metadata of the replaced
64+
/// columns.
65+
///
66+
/// The metadata of each literal, such as an Arrow extension type, is taken from the
67+
/// field with the same name in `schema`. Expressions that read the field metadata of
68+
/// their arguments, such as `arrow_metadata`, then see the same metadata as for the
69+
/// column.
70+
pub fn replace_columns_with_literals_keeping_metadata<K, V>(
71+
expr: Arc<dyn PhysicalExpr>,
72+
replacements: &HashMap<K, V>,
73+
schema: &Schema,
74+
) -> Result<Arc<dyn PhysicalExpr>>
75+
where
76+
K: Borrow<str> + Eq + Hash,
77+
V: Borrow<ScalarValue>,
78+
{
79+
replace_columns_with_literals_impl(expr, replacements, Some(schema))
80+
}
81+
82+
fn replace_columns_with_literals_impl<K, V>(
83+
expr: Arc<dyn PhysicalExpr>,
84+
replacements: &HashMap<K, V>,
85+
schema: Option<&Schema>,
86+
) -> Result<Arc<dyn PhysicalExpr>>
5587
where
5688
K: Borrow<str> + Eq + Hash,
5789
V: Borrow<ScalarValue>,
@@ -60,9 +92,14 @@ where
6092
if let Some(column) = expr.downcast_ref::<Column>()
6193
&& let Some(replacement_value) = replacements.get(column.name())
6294
{
63-
return Ok(Transformed::yes(expressions::lit(
95+
let metadata = schema
96+
.and_then(|schema| schema.field_with_name(column.name()).ok())
97+
.map(FieldMetadata::from)
98+
.filter(|metadata| !metadata.is_empty());
99+
return Ok(Transformed::yes(Arc::new(Literal::new_with_metadata(
64100
replacement_value.borrow().clone(),
65-
)));
101+
metadata,
102+
))));
66103
}
67104
Ok(Transformed::no(expr))
68105
})
@@ -924,7 +961,7 @@ mod tests {
924961
use arrow::datatypes::{Field, Fields, Schema};
925962
use datafusion_common::assert_contains;
926963
use datafusion_expr::Operator;
927-
use datafusion_physical_expr::expressions::{Column, Literal, col};
964+
use datafusion_physical_expr::expressions::{self, Column, Literal, col};
928965

929966
fn assert_cast_expr(expr: &Arc<dyn PhysicalExpr>) -> &CastExpr {
930967
expr.downcast_ref::<CastExpr>().expect("Expected CastExpr")
@@ -1262,6 +1299,37 @@ mod tests {
12621299
Ok(())
12631300
}
12641301

1302+
#[test]
1303+
fn test_replace_columns_with_literals_keeping_metadata() -> Result<()> {
1304+
let value = ScalarValue::Int64(Some(10));
1305+
let replacements = HashMap::from([("a", &value), ("b", &value)]);
1306+
let metadata =
1307+
HashMap::from([("ARROW:extension:name".to_string(), "example".to_string())]);
1308+
let schema = Schema::new(vec![
1309+
Field::new("a", DataType::Int64, true).with_metadata(metadata.clone()),
1310+
Field::new("b", DataType::Int64, true),
1311+
]);
1312+
1313+
let literal_metadata = |name: &str, index: usize| -> Result<_> {
1314+
let column_expr = Arc::new(Column::new(name, index)) as Arc<dyn PhysicalExpr>;
1315+
let result = replace_columns_with_literals_keeping_metadata(
1316+
column_expr,
1317+
&replacements,
1318+
&schema,
1319+
)?;
1320+
let literal = result
1321+
.downcast_ref::<Literal>()
1322+
.expect("Expected literal expression");
1323+
assert_eq!(*literal.value(), value);
1324+
Ok(literal.return_field(&schema)?.metadata().clone())
1325+
};
1326+
1327+
assert_eq!(literal_metadata("a", 0)?, metadata);
1328+
assert!(literal_metadata("b", 1)?.is_empty());
1329+
1330+
Ok(())
1331+
}
1332+
12651333
#[test]
12661334
fn test_replace_columns_with_literals_no_match() -> Result<()> {
12671335
let value = ScalarValue::Utf8(Some("test_value".to_string()));

‎datafusion/sqllogictest/test_files/cast_extension_type_metadata.slt‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,3 +84,31 @@ FROM (
8484
);
8585
----
8686
00010203040506070809000102030506 NULL
87+
88+
# Extension types of Parquet columns are visible to expressions evaluated in the scan,
89+
# also when the scan replaces a constant column with a literal from the statistics
90+
statement ok
91+
COPY (
92+
SELECT CAST(arrow_cast(X'00010203040506070809000102030506', 'FixedSizeBinary(16)') AS UUID) AS uuid_val
93+
)
94+
TO 'test_files/scratch/cast_extension_type_metadata/uuid.parquet'
95+
STORED AS PARQUET;
96+
97+
statement ok
98+
SET datafusion.execution.parquet.skip_metadata = false;
99+
100+
statement ok
101+
CREATE EXTERNAL TABLE uuid_parquet
102+
STORED AS PARQUET
103+
LOCATION 'test_files/scratch/cast_extension_type_metadata/uuid.parquet';
104+
105+
query ?T
106+
SELECT uuid_val, arrow_metadata(uuid_val, 'ARROW:extension:name') FROM uuid_parquet;
107+
----
108+
00010203040506070809000102030506 arrow.uuid
109+
110+
statement ok
111+
DROP TABLE uuid_parquet;
112+
113+
statement ok
114+
RESET datafusion.execution.parquet.skip_metadata;

0 commit comments

Comments
 (0)