Skip to content

Commit 3411744

Browse files
committed
Address native array extrema review feedback
1 parent d3fd897 commit 3411744

11 files changed

Lines changed: 302 additions & 55 deletions

File tree

‎native/core/src/execution/jni_api.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2341,7 +2341,6 @@ mod tests {
23412341
}],
23422342
return_type: None,
23432343
fail_on_error: false,
2344-
..Default::default()
23452344
})),
23462345
query_context: None,
23472346
expr_id: None,

‎native/core/src/execution/planner.rs‎

Lines changed: 72 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -893,6 +893,23 @@ impl PhysicalPlanner {
893893
expr.ordinal as usize,
894894
)))
895895
}
896+
ExprStruct::ArrayExtrema(expr) => {
897+
let child =
898+
self.create_expr(expr.child.as_ref().unwrap(), Arc::clone(&input_schema))?;
899+
let udf = Arc::new(ScalarUDF::from(SparkArrayExtrema::with_collations(
900+
expr.is_min,
901+
&expr.string_collations,
902+
expr.collation_unicode_version,
903+
)?));
904+
let data_type = udf.return_type(&[child.data_type(&input_schema)?])?;
905+
Ok(Arc::new(ScalarFunctionExpr::new(
906+
udf.name(),
907+
Arc::clone(&udf),
908+
vec![child],
909+
Arc::new(Field::new(udf.name(), data_type, true)),
910+
Arc::new(ConfigOptions::default()),
911+
)))
912+
}
896913
ExprStruct::ArrayInsert(expr) => {
897914
let src_array_expr = self.create_expr(
898915
expr.src_array_expr.as_ref().unwrap(),
@@ -3802,20 +3819,12 @@ impl PhysicalPlanner {
38023819
}
38033820
};
38043821

3805-
let fun_expr = if matches!(fun_name.as_str(), "array_min" | "array_max") {
3806-
Arc::new(ScalarUDF::from(SparkArrayExtrema::with_collations(
3807-
fun_name == "array_min",
3808-
&expr.string_collations,
3809-
expr.collation_unicode_version,
3810-
)?))
3811-
} else {
3812-
create_comet_physical_fun(
3813-
fun_name,
3814-
data_type.clone(),
3815-
&self.session_ctx.state(),
3816-
Some(expr.fail_on_error),
3817-
)?
3818-
};
3822+
let fun_expr = create_comet_physical_fun(
3823+
fun_name,
3824+
data_type.clone(),
3825+
&self.session_ctx.state(),
3826+
Some(expr.fail_on_error),
3827+
)?;
38193828

38203829
let args = args
38213830
.into_iter()
@@ -5213,6 +5222,55 @@ mod tests {
52135222
max_frame_size: usize,
52145223
}
52155224

5225+
#[test]
5226+
fn array_extrema_proto_preserves_mode_and_collation() {
5227+
let values = ListArray::new(
5228+
Arc::new(Field::new("item", DataType::Utf8, true)),
5229+
arrow::buffer::OffsetBuffer::new(vec![0, 2].into()),
5230+
Arc::new(StringArray::from(vec!["a", "B"])),
5231+
None,
5232+
);
5233+
let schema = Arc::new(Schema::new(vec![Field::new(
5234+
"values",
5235+
values.data_type().clone(),
5236+
true,
5237+
)]));
5238+
let batch = RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(values)]).unwrap();
5239+
let planner = PhysicalPlanner::default();
5240+
5241+
for (is_min, collations, expected) in [
5242+
(true, vec![], "B"),
5243+
(false, vec![], "a"),
5244+
(true, vec!["UTF8_LCASE".to_string()], "a"),
5245+
(false, vec!["UTF8_LCASE".to_string()], "B"),
5246+
] {
5247+
let expr = Expr {
5248+
expr_struct: Some(ExprStruct::ArrayExtrema(Box::new(
5249+
spark_expression::ArrayExtrema {
5250+
child: Some(Box::new(Expr {
5251+
expr_struct: Some(Bound(spark_expression::BoundReference {
5252+
index: 0,
5253+
datatype: None,
5254+
})),
5255+
..Default::default()
5256+
})),
5257+
is_min,
5258+
string_collations: collations,
5259+
collation_unicode_version: 16,
5260+
},
5261+
))),
5262+
..Default::default()
5263+
};
5264+
let physical = planner.create_expr(&expr, Arc::clone(&schema)).unwrap();
5265+
assert_eq!(physical.data_type(&schema).unwrap(), DataType::Utf8);
5266+
let result = physical.evaluate(&batch).unwrap().into_array(1).unwrap();
5267+
assert_eq!(
5268+
ScalarValue::try_from_array(&result, 0).unwrap(),
5269+
ScalarValue::Utf8(Some(expected.to_string()))
5270+
);
5271+
}
5272+
}
5273+
52165274
#[test]
52175275
fn scan_default_rejects_struct_expressions() {
52185276
let planner = PhysicalPlanner::new(Arc::new(SessionContext::new()), 0);
@@ -6578,7 +6636,6 @@ mod tests {
65786636
args: vec![array_col, array_col_1],
65796637
return_type: None,
65806638
fail_on_error: false,
6581-
..Default::default()
65826639
})),
65836640
query_context: None,
65846641
expr_id: None,
@@ -6705,7 +6762,6 @@ mod tests {
67056762
args: vec![array_col, array_col_1],
67066763
return_type: None,
67076764
fail_on_error: false,
6708-
..Default::default()
67096765
})),
67106766
query_context: None,
67116767
expr_id: None,

‎native/core/src/execution/planner/expression_registry.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ pub enum ExpressionType {
9797
ToPrettyString,
9898
ListExtract,
9999
GetArrayStructFields,
100+
ArrayExtrema,
100101
ArrayInsert,
101102
Rand,
102103
Randn,
@@ -374,6 +375,7 @@ impl ExpressionRegistry {
374375
Some(ExprStruct::ToPrettyString(_)) => Ok(ExpressionType::ToPrettyString),
375376
Some(ExprStruct::ListExtract(_)) => Ok(ExpressionType::ListExtract),
376377
Some(ExprStruct::GetArrayStructFields(_)) => Ok(ExpressionType::GetArrayStructFields),
378+
Some(ExprStruct::ArrayExtrema(_)) => Ok(ExpressionType::ArrayExtrema),
377379
Some(ExprStruct::ArrayInsert(_)) => Ok(ExpressionType::ArrayInsert),
378380
Some(ExprStruct::Rand(_)) => Ok(ExpressionType::Rand),
379381
Some(ExprStruct::Randn(_)) => Ok(ExpressionType::Randn),

‎native/proto/src/proto/expr.proto‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ message Expr {
9494
Shuffle shuffle = 72;
9595
RandStr rand_str = 73;
9696
Uuid uuid = 74;
97+
ArrayExtrema array_extrema = 75;
9798
}
9899

99100
reserved 20;
@@ -540,10 +541,6 @@ message ScalarFunc {
540541
repeated Expr args = 2;
541542
DataType return_type = 3;
542543
bool fail_on_error = 4;
543-
// array_min/max only: collations of every string leaf in the element type,
544-
// visiting array elements and struct fields depth-first. Empty means binary ordering.
545-
repeated string string_collations = 5;
546-
uint32 collation_unicode_version = 6;
547544
}
548545

549546
message CaseWhen {
@@ -630,6 +627,15 @@ enum NullOrdering {
630627
}
631628

632629
// Array functions
630+
message ArrayExtrema {
631+
Expr child = 1;
632+
bool is_min = 2;
633+
// Collations of every string leaf in the element type, visiting array elements
634+
// and struct fields depth-first. Empty means binary ordering.
635+
repeated string string_collations = 3;
636+
uint32 collation_unicode_version = 4;
637+
}
638+
633639
message ArrayInsert {
634640
Expr src_array_expr = 1;
635641
Expr pos_expr = 2;

‎native/spark-expr/benches/array_extrema.rs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,31 @@ fn criterion_benchmark(c: &mut Criterion) {
197197
group.bench_function(BenchmarkId::new("lcase_min", shape), |b| {
198198
b.iter(|| black_box(comet.invoke_with_args(black_box(args.clone())).unwrap()))
199199
});
200+
201+
for is_min in [true, false] {
202+
let comet = ScalarUDF::from(SparkArrayExtrema::new(is_min));
203+
let datafusion = if is_min {
204+
array_min_udf()
205+
} else {
206+
array_max_udf()
207+
};
208+
let evaluate = |udf: &ScalarUDF| {
209+
udf.invoke_with_args(args.clone())
210+
.unwrap()
211+
.into_array(input.len())
212+
.unwrap()
213+
};
214+
assert_eq!(evaluate(&comet).to_data(), evaluate(&datafusion).to_data());
215+
let op = if is_min { "min" } else { "max" };
216+
for (engine, udf) in [("comet", &comet), ("datafusion", datafusion.as_ref())] {
217+
group.bench_function(
218+
BenchmarkId::new(format!("binary_{op}_{engine}"), shape),
219+
|b| {
220+
b.iter(|| black_box(udf.invoke_with_args(black_box(args.clone())).unwrap()))
221+
},
222+
);
223+
}
224+
}
200225
}
201226
group.finish();
202227
}

‎native/spark-expr/src/array_funcs/array_extrema.rs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,13 @@ impl ScalarUDFImpl for SparkArrayExtrema {
164164
if self.string_collations.is_empty()
165165
&& !matches!(
166166
element_type,
167-
DataType::Float32 | DataType::Float64 | DataType::List(_) | DataType::Struct(_)
167+
DataType::Float32
168+
| DataType::Float64
169+
| DataType::Utf8
170+
| DataType::LargeUtf8
171+
| DataType::Utf8View
172+
| DataType::List(_)
173+
| DataType::Struct(_)
168174
)
169175
{
170176
return self.datafusion_udf.invoke_with_args(args);
@@ -274,6 +280,10 @@ fn nested_extrema(
274280

275281
fn take_extrema_values(values: &ArrayRef, indices: &UInt32Array) -> Result<ArrayRef> {
276282
match values.data_type() {
283+
DataType::Utf8View => {
284+
let result = take(values.as_ref(), indices, None)?;
285+
Ok(Arc::new(result.as_string_view().gc()))
286+
}
277287
// Arrow's flat-list take is faster, but its child capacity estimate can
278288
// grow excessively for sparse outputs or recursively nested children.
279289
DataType::List(field)

0 commit comments

Comments
 (0)