Skip to content

Commit b61236f

Browse files
committed
address pr feeback
- simplify LambdaArgument::evaluate to improve readability - rename MultiArgHigherOrderUdf to MultiArgHigherOrderUDF to keep consistency with the rest of the codebase - cache NullArray in HigherOrderFunctionExpr::evaluate instead of creating a new one for each uncaptured column
1 parent b1f83e8 commit b61236f

3 files changed

Lines changed: 49 additions & 33 deletions

File tree

‎datafusion/expr/src/higher_order_function.rs‎

Lines changed: 13 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -348,32 +348,29 @@ impl LambdaArgument {
348348
exec_datafusion_err!("lambdas with zero parameters are not supported")
349349
})?;
350350

351-
let spread_captures = self
352-
.captures
353-
.as_ref()
354-
.filter(|b| !b.columns().is_empty())
355-
.map(|captures| {
356-
let spread_columns = if captures
351+
let columns = match &self.captures {
352+
Some(captures) if !captures.columns().is_empty() => {
353+
if captures
357354
.schema_ref()
358355
.fields()
359356
.iter()
360357
.all(|f| f.data_type().is_null())
361358
{
362359
let null = Arc::new(NullArray::new(row_count)) as _;
363360

364-
vec![null; captures.num_columns()]
361+
std::iter::repeat_n(null, captures.num_columns())
362+
.chain(args)
363+
.collect()
365364
} else {
366365
spread_captures(captures.columns())?
367-
};
368-
369-
RecordBatch::try_new(captures.schema(), spread_columns)
370-
})
371-
.transpose()?;
372-
373-
let columns = match spread_captures {
374-
Some(captures) => captures.columns().iter().cloned().chain(args).collect(),
375-
None => args,
366+
.into_iter()
367+
.chain(args)
368+
.collect()
369+
}
370+
}
371+
_ => args,
376372
};
373+
377374
let merged = RecordBatch::try_new(Arc::clone(&self.schema), columns)?;
378375

379376
self.body.evaluate(&merged)

‎datafusion/physical-expr/src/higher_order_function.rs‎

Lines changed: 33 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -337,20 +337,40 @@ impl PhysicalExpr for HigherOrderFunctionExpr {
337337
.map(|(name, param)| param.renamed(name.as_str()))
338338
.collect();
339339

340-
let (fields, columns): (Vec<_>, _) = std::iter::zip(batch.schema_ref().fields(), batch.columns())
341-
.enumerate()
342-
.map(|(i, (field, array))| if lambda.used_column_indices().contains(&i) {
343-
(Arc::clone(field), Arc::clone(array))
344-
} else {
345-
(Arc::new(Field::new(field.name(), DataType::Null, true)), Arc::new(NullArray::new(batch.num_rows())) as _)
346-
})
347-
.collect();
348-
349-
let batch= RecordBatch::try_new_with_options(
340+
let mut null_array = None;
341+
let mut null_field = None;
342+
343+
let (fields, columns): (Vec<_>, _) =
344+
std::iter::zip(batch.schema_ref().fields(), batch.columns())
345+
.enumerate()
346+
.map(|(i, (field, array))| {
347+
if lambda.used_column_indices().contains(&i) {
348+
(Arc::clone(field), Arc::clone(array))
349+
} else {
350+
let null_field =
351+
null_field.get_or_insert_with(|| {
352+
Arc::new(Field::new(
353+
field.name(),
354+
DataType::Null,
355+
true,
356+
))
357+
});
358+
359+
let null_array =
360+
null_array.get_or_insert_with(|| {
361+
Arc::new(NullArray::new(batch.num_rows()))
362+
});
363+
364+
(Arc::clone(null_field), Arc::clone(null_array) as _)
365+
}
366+
})
367+
.collect();
368+
369+
let batch = RecordBatch::try_new_with_options(
350370
Arc::new(Schema::new(fields)),
351371
columns,
352-
&RecordBatchOptions::new().with_row_count(Some(batch.num_rows())),
353-
)?;
372+
&RecordBatchOptions::new().with_row_count(Some(batch.num_rows())),
373+
)?;
354374

355375
Ok(ValueOrLambda::Lambda(LambdaArgument::new(
356376
params,
@@ -369,8 +389,7 @@ impl PhysicalExpr for HigherOrderFunctionExpr {
369389
&& matches!(
370390
value.data_type(),
371391
DataType::List(_) | DataType::LargeList(_)
372-
)
373-
{
392+
) {
374393
let arr = value.into_array(batch.num_rows())?;
375394
if arr.null_count() == 0 {
376395
ColumnarValue::Array(arr)

‎datafusion/sqllogictest/src/test_context.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -725,11 +725,11 @@ fn register_multi_arg_lambda_udf(ctx: &SessionContext) {
725725
// a dummy udf that provides every value argument it receives as a parameter to it's lambda, the last argument:
726726
// multi_arg_higher_order_udf([..value_args], ([..value_args_as_params]) -> ...)
727727
#[derive(Debug, PartialEq, Eq, Hash)]
728-
struct MultiArgHigherOrderUdf {
728+
struct MultiArgHigherOrderUDF {
729729
signature: HigherOrderSignature,
730730
}
731731

732-
impl HigherOrderUDFImpl for MultiArgHigherOrderUdf {
732+
impl HigherOrderUDFImpl for MultiArgHigherOrderUDF {
733733
fn name(&self) -> &str {
734734
"multi_arg_higher_order_udf"
735735
}
@@ -801,7 +801,7 @@ fn register_multi_arg_lambda_udf(ctx: &SessionContext) {
801801
}
802802

803803
ctx.register_higher_order_function(Arc::new(HigherOrderUDF::new_from_impl(
804-
MultiArgHigherOrderUdf {
804+
MultiArgHigherOrderUDF {
805805
signature: HigherOrderSignature::variadic_any(Volatility::Immutable),
806806
},
807807
)));

0 commit comments

Comments
 (0)