Skip to content

Commit d05bc23

Browse files
committed
Carry logical CASE result metadata through physical lowering
1 parent 185ae25 commit d05bc23

3 files changed

Lines changed: 145 additions & 7 deletions

File tree

‎datafusion/physical-expr/src/expressions/case.rs‎

Lines changed: 62 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ use arrow::datatypes::{DataType, Field, FieldRef, Schema, UInt32Type, UnionMode}
3232
use arrow::error::ArrowError;
3333
use arrow_schema::extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY};
3434
use datafusion_common::cast::as_boolean_array;
35+
use datafusion_common::metadata::FieldMetadata;
3536
use datafusion_common::{
3637
DataFusionError, Result, ScalarValue, assert_or_internal_err, exec_err,
3738
internal_datafusion_err, internal_err,
@@ -278,6 +279,8 @@ pub struct CaseExpr {
278279
body: CaseBody,
279280
/// Evaluation method to use
280281
eval_method: EvalMethod,
282+
/// Logical result type and metadata inferred before physical lowering.
283+
logical_result_field: Option<(DataType, FieldMetadata)>,
281284
}
282285

283286
// eval_method is functionally derived from body, so excluding it from
@@ -286,12 +289,13 @@ pub struct CaseExpr {
286289
impl Hash for CaseExpr {
287290
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
288291
self.body.hash(state);
292+
self.logical_result_field.hash(state);
289293
}
290294
}
291295

292296
impl PartialEq for CaseExpr {
293297
fn eq(&self, other: &Self) -> bool {
294-
self.body == other.body
298+
self.body == other.body && self.logical_result_field == other.logical_result_field
295299
}
296300
}
297301

@@ -678,7 +682,20 @@ impl CaseExpr {
678682

679683
let eval_method = Self::find_best_eval_method(&body)?;
680684

681-
Ok(Self { body, eval_method })
685+
Ok(Self {
686+
body,
687+
eval_method,
688+
logical_result_field: None,
689+
})
690+
}
691+
692+
/// Preserve the logical CASE result metadata across physical lowering.
693+
pub fn with_logical_result_field(mut self, field: &FieldRef) -> Self {
694+
self.logical_result_field = Some((
695+
field.data_type().clone(),
696+
FieldMetadata::from(field.metadata().clone()),
697+
));
698+
self
682699
}
683700

684701
fn find_best_eval_method(body: &CaseBody) -> Result<EvalMethod> {
@@ -1439,6 +1456,17 @@ impl PhysicalExpr for CaseExpr {
14391456
fn return_field(&self, input_schema: &Schema) -> Result<FieldRef> {
14401457
let data_type = self.data_type(input_schema)?;
14411458
let nullable = self.nullable(input_schema)?;
1459+
if let Some((logical_type, logical_metadata)) = &self.logical_result_field {
1460+
let metadata = if logical_type == &data_type {
1461+
logical_metadata.to_hashmap()
1462+
} else {
1463+
Metadata::new()
1464+
};
1465+
return Ok(Arc::new(
1466+
Field::new(format!("{self}"), data_type, nullable)
1467+
.with_metadata(metadata),
1468+
));
1469+
}
14421470
if let Some((_, first_result)) = self.body.when_then_expr.first()
14431471
&& (first_result.is::<Column>() || first_result.is::<Literal>())
14441472
{
@@ -1597,11 +1625,15 @@ impl PhysicalExpr for CaseExpr {
15971625
),
15981626
(false, false) => (None, &children[0..children.len()], None),
15991627
};
1600-
Ok(Arc::new(CaseExpr::try_new(
1628+
let mut rebuilt = CaseExpr::try_new(
16011629
expr.cloned(),
16021630
when_then_expr.iter().cloned().tuples().collect(),
16031631
else_expr.cloned(),
1604-
)?))
1632+
)?;
1633+
rebuilt
1634+
.logical_result_field
1635+
.clone_from(&self.logical_result_field);
1636+
Ok(Arc::new(rebuilt))
16051637
}
16061638
}
16071639

@@ -1944,6 +1976,32 @@ mod tests {
19441976
Ok(())
19451977
}
19461978

1979+
#[test]
1980+
fn case_logical_field_survives_rewriting_children() -> Result<()> {
1981+
let metadata = HashMap::from([("source".to_string(), "physical".to_string())]);
1982+
let schema = Schema::new(vec![
1983+
Field::new("a", DataType::Binary, false).with_metadata(metadata.clone()),
1984+
Field::new("b", DataType::Binary, false).with_metadata(metadata),
1985+
]);
1986+
let expr: Arc<dyn PhysicalExpr> = Arc::new(
1987+
CaseExpr::try_new(
1988+
None,
1989+
vec![(lit(true), col("a", &schema)?)],
1990+
Some(col("b", &schema)?),
1991+
)?
1992+
.with_logical_result_field(&Arc::new(Field::new(
1993+
"logical_case",
1994+
DataType::Binary,
1995+
true,
1996+
))),
1997+
);
1998+
assert!(expr.return_field(&schema)?.metadata().is_empty());
1999+
let children = expr.children().into_iter().cloned().collect();
2000+
let rewritten = expr.with_new_children(children)?;
2001+
assert!(rewritten.return_field(&schema)?.metadata().is_empty());
2002+
Ok(())
2003+
}
2004+
19472005
#[test]
19482006
fn case_not_does_not_inherit_input_metadata() -> Result<()> {
19492007
let metadata = HashMap::from([("logical".to_string(), "boolean".to_string())]);

‎datafusion/physical-expr/src/planner.rs‎

Lines changed: 63 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
4141
use datafusion_expr::var_provider::VarType;
4242
use datafusion_expr::var_provider::is_system_variables;
4343
use datafusion_expr::{
44-
Between, BinaryExpr, Expr, Like, Operator, TryCast, binary_expr, lit,
44+
Between, BinaryExpr, Expr, ExprSchemable, Like, Operator, TryCast, binary_expr, lit,
4545
};
4646

4747
/// [PhysicalExpr] evaluate DataFusion expressions such as `A + 1`, or `CAST(c1
@@ -371,6 +371,25 @@ pub fn create_physical_expr(
371371
.zip(then_expr.iter())
372372
.map(|(w, t)| (Arc::clone(w), Arc::clone(t)))
373373
.collect();
374+
// Match the logical CASE fast path without inferring its field twice.
375+
let common_unmarked_branch =
376+
match (case.when_then_expr.first(), when_then_expr.first()) {
377+
(Some((_, logical_result)), Some((_, physical_result)))
378+
if matches!(
379+
logical_result.as_ref(),
380+
Expr::Column(_) | Expr::Literal(_, _)
381+
) =>
382+
{
383+
let field = physical_result.return_field(input_schema)?;
384+
!field.data_type().is_null()
385+
&& !matches!(
386+
logical_result.as_ref(),
387+
Expr::Literal(value, _) if value.is_null()
388+
)
389+
&& field.metadata().is_empty()
390+
}
391+
_ => false,
392+
};
374393
let else_expr: Option<Arc<dyn PhysicalExpr>> =
375394
if let Some(e) = &case.else_expr {
376395
Some(create_physical_expr(
@@ -382,7 +401,16 @@ pub fn create_physical_expr(
382401
} else {
383402
None
384403
};
385-
Ok(expressions::case(expr, when_then_expr, else_expr)?)
404+
let physical_case =
405+
expressions::CaseExpr::try_new(expr, when_then_expr, else_expr)?;
406+
if common_unmarked_branch {
407+
Ok(Arc::new(physical_case))
408+
} else {
409+
Ok(Arc::new(
410+
physical_case
411+
.with_logical_result_field(&e.to_field(input_dfschema)?.1),
412+
))
413+
}
386414
}
387415
Expr::Cast(Cast { expr, field }) => expressions::cast_with_target_field(
388416
create_physical_expr(expr, input_dfschema, execution_props, planning_ctx)?,
@@ -738,7 +766,7 @@ mod tests {
738766
use datafusion_expr::physical_planning_context::{
739767
ScalarSubqueryResults, SubqueryIndex,
740768
};
741-
use datafusion_expr::{LogicalPlanBuilder, col, scalar_subquery};
769+
use datafusion_expr::{LogicalPlanBuilder, col, scalar_subquery, when};
742770

743771
use super::*;
744772

@@ -756,6 +784,38 @@ mod tests {
756784
)
757785
}
758786

787+
#[test]
788+
fn case_metadata_matches_logical_field_after_alias_lowering() -> Result<()> {
789+
let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
790+
let df_schema = DFSchema::try_from(schema.clone())?;
791+
let mut aliased = col("a");
792+
for _ in 0..10 {
793+
aliased = aliased.alias("nested");
794+
}
795+
let marked = |arg| {
796+
datafusion_functions::core::expr_fn::with_metadata(vec![
797+
arg,
798+
lit("source"),
799+
lit("test"),
800+
])
801+
};
802+
let logical = when(lit(true), marked(aliased)).otherwise(marked(col("a")))?;
803+
let logical_field = logical.to_field(&df_schema)?.1;
804+
assert!(logical_field.metadata().is_empty());
805+
806+
let physical = create_physical_expr(
807+
&logical,
808+
&df_schema,
809+
&ExecutionProps::new(),
810+
&PhysicalPlanningContext::default(),
811+
)?;
812+
assert_eq!(
813+
physical.return_field(&schema)?.metadata(),
814+
logical_field.metadata()
815+
);
816+
Ok(())
817+
}
818+
759819
fn as_planner_cast(physical: &Arc<dyn PhysicalExpr>) -> &expressions::CastExpr {
760820
physical
761821
.downcast_ref::<expressions::CastExpr>()

‎datafusion/sqllogictest/test_files/metadata.slt‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -430,6 +430,26 @@ select arrow_metadata(with_metadata(id, 'unit', 'ms'), 'metadata_key') from tabl
430430
----
431431
the id field
432432

433+
# CASE must expose the same field metadata directly and through a projection.
434+
query T
435+
select arrow_metadata(
436+
case when id = 1 then id
437+
else with_metadata(id, 'metadata_key', 'the id field') end,
438+
'metadata_key')
439+
from table_with_metadata where id = 1;
440+
----
441+
the id field
442+
443+
query T
444+
select arrow_metadata(chosen, 'metadata_key')
445+
from (
446+
select case when id = 1 then id
447+
else with_metadata(id, 'metadata_key', 'the id field') end as chosen
448+
from table_with_metadata where id = 1
449+
);
450+
----
451+
the id field
452+
433453
# with_metadata: new keys overwrite existing on collision
434454
query T
435455
select arrow_metadata(with_metadata(id, 'metadata_key', 'overridden'), 'metadata_key') from table_with_metadata limit 1;

0 commit comments

Comments
 (0)