Skip to content

Commit e91b23e

Browse files
committed
feat: add array_exists with lambda expression support
Add native support for `array_exists(arr, x -> predicate(x))` in SQL and DataFrame API. This is the first general-purpose lambda expression infrastructure, which can later be extended to support `array_filter`, `array_transform`, and `array_forall`. The lambda body is serialized as a regular expression tree where `NamedLambdaVariable` leaf nodes are serialized as `LambdaVariable` proto messages. On the Rust side, `ArrayExistsExpr` evaluates the lambda body vectorized over all elements in a single pass: it flattens list values, expands the batch with repeat indices, appends elements as a `__comet_lambda_var` column, evaluates once, and reduces per row with SQL three-valued logic semantics. Unsupported lambda bodies (e.g. containing UDFs) fall back to Spark. Closes apache#3149
1 parent a3f7f66 commit e91b23e

10 files changed

Lines changed: 876 additions & 1 deletion

File tree

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! Array expression builders
19+
20+
use std::sync::Arc;
21+
22+
use arrow::datatypes::SchemaRef;
23+
use datafusion::physical_expr::PhysicalExpr;
24+
use datafusion_comet_proto::spark_expression::Expr;
25+
use datafusion_comet_spark_expr::{ArrayExistsExpr, LambdaVariableExpr};
26+
27+
use crate::execution::operators::ExecutionError;
28+
use crate::execution::planner::expression_registry::ExpressionBuilder;
29+
use crate::execution::planner::PhysicalPlanner;
30+
use crate::execution::serde::to_arrow_datatype;
31+
use crate::extract_expr;
32+
33+
pub struct ArrayExistsBuilder;
34+
35+
impl ExpressionBuilder for ArrayExistsBuilder {
36+
fn build(
37+
&self,
38+
spark_expr: &Expr,
39+
input_schema: SchemaRef,
40+
planner: &PhysicalPlanner,
41+
) -> Result<Arc<dyn PhysicalExpr>, ExecutionError> {
42+
let expr = extract_expr!(spark_expr, ArrayExists);
43+
let array_expr =
44+
planner.create_expr(expr.array.as_ref().unwrap(), Arc::clone(&input_schema))?;
45+
let lambda_body = planner.create_expr(expr.lambda_body.as_ref().unwrap(), input_schema)?;
46+
Ok(Arc::new(ArrayExistsExpr::new(
47+
array_expr,
48+
lambda_body,
49+
expr.follow_three_valued_logic,
50+
)))
51+
}
52+
}
53+
54+
pub struct LambdaVariableBuilder;
55+
56+
impl ExpressionBuilder for LambdaVariableBuilder {
57+
fn build(
58+
&self,
59+
spark_expr: &Expr,
60+
_input_schema: SchemaRef,
61+
_planner: &PhysicalPlanner,
62+
) -> Result<Arc<dyn PhysicalExpr>, ExecutionError> {
63+
let expr = extract_expr!(spark_expr, LambdaVariable);
64+
let data_type = to_arrow_datatype(expr.datatype.as_ref().unwrap());
65+
Ok(Arc::new(LambdaVariableExpr::new(data_type)))
66+
}
67+
}

‎native/core/src/execution/expressions/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
//! Native DataFusion expressions
1919
2020
pub mod arithmetic;
21+
pub mod array;
2122
pub mod bitwise;
2223
pub mod comparison;
2324
pub mod logical;

‎native/core/src/execution/planner/expression_registry.rs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,8 @@ pub enum ExpressionType {
103103
Randn,
104104
SparkPartitionId,
105105
MonotonicallyIncreasingId,
106+
ArrayExists,
107+
LambdaVariable,
106108

107109
// Time functions
108110
Hour,
@@ -184,6 +186,9 @@ impl ExpressionRegistry {
184186

185187
// Register temporal expressions
186188
self.register_temporal_expressions();
189+
190+
// Register array expressions
191+
self.register_array_expressions();
187192
}
188193

189194
/// Register arithmetic expression builders
@@ -306,6 +311,18 @@ impl ExpressionRegistry {
306311
);
307312
}
308313

314+
/// Register array expression builders
315+
fn register_array_expressions(&mut self) {
316+
use crate::execution::expressions::array::*;
317+
318+
self.builders
319+
.insert(ExpressionType::ArrayExists, Box::new(ArrayExistsBuilder));
320+
self.builders.insert(
321+
ExpressionType::LambdaVariable,
322+
Box::new(LambdaVariableBuilder),
323+
);
324+
}
325+
309326
/// Extract expression type from Spark protobuf expression
310327
fn get_expression_type(spark_expr: &Expr) -> Result<ExpressionType, ExecutionError> {
311328
match spark_expr.expr_struct.as_ref() {
@@ -370,6 +387,8 @@ impl ExpressionRegistry {
370387
Some(ExprStruct::MonotonicallyIncreasingId(_)) => {
371388
Ok(ExpressionType::MonotonicallyIncreasingId)
372389
}
390+
Some(ExprStruct::ArrayExists(_)) => Ok(ExpressionType::ArrayExists),
391+
Some(ExprStruct::LambdaVariable(_)) => Ok(ExpressionType::LambdaVariable),
373392

374393
Some(ExprStruct::Hour(_)) => Ok(ExpressionType::Hour),
375394
Some(ExprStruct::Minute(_)) => Ok(ExpressionType::Minute),

‎native/proto/src/proto/expr.proto‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,8 @@ message Expr {
8888
UnixTimestamp unix_timestamp = 65;
8989
FromJson from_json = 66;
9090
ToCsv to_csv = 67;
91+
ArrayExists array_exists = 68;
92+
LambdaVariable lambda_variable = 69;
9193
}
9294
}
9395

@@ -440,3 +442,14 @@ message ArrayJoin {
440442
message Rand {
441443
int64 seed = 1;
442444
}
445+
446+
message ArrayExists {
447+
Expr array = 1;
448+
Expr lambda_body = 2;
449+
DataType element_type = 3;
450+
bool follow_three_valued_logic = 4;
451+
}
452+
453+
message LambdaVariable {
454+
DataType datatype = 1;
455+
}

0 commit comments

Comments
 (0)