Skip to content

Commit 830ee84

Browse files
committed
Bound physical CASE metadata inference for marked functions
1 parent 816119e commit 830ee84

1 file changed

Lines changed: 95 additions & 3 deletions

File tree

  • datafusion/physical-expr/src/expressions

‎datafusion/physical-expr/src/expressions/case.rs‎

Lines changed: 95 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ use crate::PhysicalExpr;
2222
use crate::expressions::{
2323
CastExpr, LambdaVariable, NegativeExpr, NotExpr, TryCastExpr, lit, try_cast,
2424
};
25+
use crate::{HigherOrderFunctionExpr, ScalarFunctionExpr};
2526
use arrow::array::*;
2627
use arrow::compute::kernels::zip::zip;
2728
use arrow::compute::{
@@ -1256,6 +1257,21 @@ fn case_result_field_metadata(
12561257
case: &CaseExpr,
12571258
input_schema: &Schema,
12581259
) -> Result<(DataType, Metadata)> {
1260+
const MAX_EXACT_BRANCH_DEPTH: usize = 8;
1261+
1262+
fn can_infer_exact_function_branch(expr: &dyn PhysicalExpr) -> bool {
1263+
let mut work = vec![(expr, 1)];
1264+
while let Some((node, depth)) = work.pop() {
1265+
if depth > MAX_EXACT_BRANCH_DEPTH {
1266+
return false;
1267+
}
1268+
for child in node.children() {
1269+
work.push((child.as_ref(), depth + 1));
1270+
}
1271+
}
1272+
true
1273+
}
1274+
12591275
struct BranchField {
12601276
data_type: DataType,
12611277
metadata: Metadata,
@@ -1311,13 +1327,28 @@ fn case_result_field_metadata(
13111327
}
13121328
} else if let Some(negative) = expr.downcast_ref::<NegativeExpr>() {
13131329
work.push(Work::Visit(negative.arg().as_ref()));
1314-
} else if let Some(not) = expr.downcast_ref::<NotExpr>() {
1315-
let field = not.return_field(input_schema)?;
1330+
} else if expr.is::<NotExpr>() {
13161331
fields.push(BranchField {
1317-
data_type: field.data_type().clone(),
1332+
data_type: DataType::Boolean,
13181333
metadata: Metadata::new(),
13191334
certainly_null: false,
13201335
});
1336+
} else if expr.is::<ScalarFunctionExpr>()
1337+
|| expr.is::<HigherOrderFunctionExpr>()
1338+
{
1339+
let field = expr.return_field(input_schema)?;
1340+
let metadata = if field.metadata().is_empty()
1341+
|| can_infer_exact_function_branch(expr)
1342+
{
1343+
field.metadata().clone()
1344+
} else {
1345+
Metadata::new()
1346+
};
1347+
fields.push(BranchField {
1348+
data_type: field.data_type().clone(),
1349+
metadata,
1350+
certainly_null: false,
1351+
});
13211352
} else {
13221353
let field = expr.return_field(input_schema)?;
13231354
fields.push(BranchField {
@@ -1759,6 +1790,7 @@ mod tests {
17591790

17601791
std::thread_local! {
17611792
static COUNTED_COLUMN_DISPLAYS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
1793+
static COUNTED_COLUMN_FIELDS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
17621794
}
17631795

17641796
#[derive(Debug, Hash, PartialEq, Eq)]
@@ -1785,6 +1817,7 @@ mod tests {
17851817
}
17861818

17871819
fn return_field(&self, input_schema: &Schema) -> Result<FieldRef> {
1820+
COUNTED_COLUMN_FIELDS.with(|count| count.set(count.get() + 1));
17881821
self.0.return_field(input_schema)
17891822
}
17901823

@@ -1921,6 +1954,24 @@ mod tests {
19211954
let negated: Arc<dyn PhysicalExpr> = Arc::new(NotExpr::new(col("a", &schema)?));
19221955
let expr = case(None, vec![(lit(true), negated)], Some(col("b", &schema)?))?;
19231956
assert!(expr.return_field(&schema)?.metadata().is_empty());
1957+
1958+
let column: Arc<dyn PhysicalExpr> = Arc::new(CountedColumn(Column::new("a", 0)));
1959+
for depth in [16, 64] {
1960+
let mut nested = Arc::clone(&column);
1961+
for _ in 0..depth {
1962+
nested = case(
1963+
None,
1964+
vec![(lit(true), Arc::new(NotExpr::new(nested)))],
1965+
Some(col("b", &schema)?),
1966+
)?;
1967+
}
1968+
COUNTED_COLUMN_FIELDS.with(|count| count.set(0));
1969+
let nested_case = nested.downcast_ref::<CaseExpr>().ok_or_else(|| {
1970+
internal_datafusion_err!("Expected nested CASE expression")
1971+
})?;
1972+
super::case_result_field_metadata(nested_case, &schema)?;
1973+
assert_eq!(COUNTED_COLUMN_FIELDS.with(|count| count.get()), 0);
1974+
}
19241975
Ok(())
19251976
}
19261977

@@ -1965,6 +2016,47 @@ mod tests {
19652016
Ok(())
19662017
}
19672018

2019+
#[test]
2020+
fn case_function_metadata_respects_logical_depth_limit() -> Result<()> {
2021+
let metadata = HashMap::from([("logical".to_string(), "binary".to_string())]);
2022+
let schema = Schema::new(vec![
2023+
Field::new("a", DataType::Binary, false).with_metadata(metadata.clone()),
2024+
Field::new("b", DataType::Binary, false).with_metadata(metadata.clone()),
2025+
]);
2026+
let udf = Arc::new(datafusion_expr::expr_fn::create_udf(
2027+
"marked_identity",
2028+
vec![DataType::Binary],
2029+
DataType::Binary,
2030+
datafusion_expr::Volatility::Immutable,
2031+
Arc::new(|args| Ok(args[0].clone())),
2032+
));
2033+
let function_field: FieldRef = Arc::new(
2034+
Field::new("marked_identity", DataType::Binary, true)
2035+
.with_metadata(metadata.clone()),
2036+
);
2037+
let options = Arc::new(datafusion_common::config::ConfigOptions::default());
2038+
for (depth, expected_marked) in [(2, true), (9, false)] {
2039+
let mut branch = col("a", &schema)?;
2040+
for _ in 0..depth {
2041+
branch = Arc::new(ScalarFunctionExpr::new(
2042+
"marked_identity",
2043+
Arc::clone(&udf),
2044+
vec![branch],
2045+
Arc::clone(&function_field),
2046+
Arc::clone(&options),
2047+
));
2048+
}
2049+
let expr = case(None, vec![(lit(true), branch)], Some(col("b", &schema)?))?;
2050+
let field = expr.return_field(&schema)?;
2051+
if expected_marked {
2052+
assert_eq!(field.metadata(), &metadata);
2053+
} else {
2054+
assert!(field.metadata().is_empty());
2055+
}
2056+
}
2057+
Ok(())
2058+
}
2059+
19682060
#[test]
19692061
fn nested_case_metadata_field_names_scale_linearly() -> Result<()> {
19702062
let metadata = HashMap::from([("logical".to_string(), "binary".to_string())]);

0 commit comments

Comments
 (0)