Skip to content

Commit bf90875

Browse files
compheadkosiew
authored andcommitted
feat: support ansi for elt (apache#23928)
Part of apache#23929 ## Rationale for this change Spark's `elt` errors on out-of-range indices when ANSI is on, and returns `NULL` when off. DataFusion previously always returned `NULL` (there was a `TODO` for this in the source). ## What changes are included in this PR? - `elt.rs`: read `enable_ansi_mode` from `ScalarFunctionArgs::config_options`; raise `"The index N is out of bounds. The array has M elements."` in ANSI mode, keep `NULL` behavior otherwise. `NULL` index always returns `NULL`. - `elt.slt`: add missing ANSI-off cases (index 0, negative, large, `NULL` index, `NULL` value, vectorized) and a full ANSI-on section with `statement error` assertions. ## Are these changes tested? Yes. New Rust unit tests and sqllogictest cases cover both modes. ## Are there any user-facing changes? With `enable_ansi_mode = true`, `elt` on invalid indices now errors instead of returning `NULL`. Default behavior unchanged. No API changes.
1 parent 74e6583 commit bf90875

2 files changed

Lines changed: 163 additions & 23 deletions

File tree

  • datafusion
    • spark/src/function/string
    • sqllogictest/test_files/spark/string

‎datafusion/spark/src/function/string/elt.rs‎

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ use arrow::compute::{can_cast_types, cast};
2424
use arrow::datatypes::DataType::{Int64, Utf8};
2525
use arrow::datatypes::{DataType, Int64Type};
2626
use datafusion_common::cast::as_string_array;
27-
use datafusion_common::{DataFusionError, Result, plan_datafusion_err};
27+
use datafusion_common::{DataFusionError, Result, exec_err, plan_datafusion_err};
2828
use datafusion_expr::{
2929
ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
3030
};
@@ -63,7 +63,11 @@ impl ScalarUDFImpl for SparkElt {
6363
}
6464

6565
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
66-
make_scalar_function(elt, vec![])(&args.args)
66+
let enable_ansi_mode = args.config_options.execution.enable_ansi_mode;
67+
make_scalar_function(
68+
move |arrays: &[ArrayRef]| elt(arrays, enable_ansi_mode),
69+
vec![],
70+
)(&args.args)
6771
}
6872

6973
fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
@@ -80,18 +84,13 @@ impl ScalarUDFImpl for SparkElt {
8084
"ELT index must be Int64 (or castable to Int64), got {idx_dt:?}"
8185
)));
8286
}
83-
let mut coerced = Vec::with_capacity(arg_types.len());
84-
coerced.push(Int64);
85-
86-
for _ in 1..length {
87-
coerced.push(Utf8);
88-
}
89-
87+
let mut coerced = vec![Utf8; length];
88+
coerced[0] = Int64;
9089
Ok(coerced)
9190
}
9291
}
9392

94-
fn elt(args: &[ArrayRef]) -> Result<ArrayRef, DataFusionError> {
93+
fn elt(args: &[ArrayRef], enable_ansi_mode: bool) -> Result<ArrayRef> {
9594
let n_rows = args[0].len();
9695

9796
let idx: &PrimitiveArray<Int64Type> =
@@ -103,11 +102,10 @@ fn elt(args: &[ArrayRef]) -> Result<ArrayRef, DataFusionError> {
103102
})?;
104103

105104
let num_values = args.len() - 1;
106-
let mut cols: Vec<Arc<StringArray>> = Vec::with_capacity(num_values);
105+
let mut cols: Vec<StringArray> = Vec::with_capacity(num_values);
107106
for a in args.iter().skip(1) {
108107
let casted = cast(a, &Utf8)?;
109-
let sa = as_string_array(&casted)?;
110-
cols.push(Arc::new(sa.clone()));
108+
cols.push(as_string_array(&casted)?.clone());
111109
}
112110

113111
let mut builder = StringBuilder::new();
@@ -120,10 +118,12 @@ fn elt(args: &[ArrayRef]) -> Result<ArrayRef, DataFusionError> {
120118

121119
let index = idx.value(i);
122120

123-
// TODO: if spark.sql.ansi.enabled is true,
124-
// throw ArrayIndexOutOfBoundsException for invalid indices;
125-
// if false, return NULL instead (current behavior).
126121
if index < 1 || (index as usize) > num_values {
122+
if enable_ansi_mode {
123+
return exec_err!(
124+
"The index {index} is out of bounds. The array has {num_values} elements."
125+
);
126+
}
127127
builder.append_null();
128128
continue;
129129
}
@@ -146,13 +146,13 @@ mod tests {
146146
use super::*;
147147
use arrow::array::Int64Array;
148148

149-
fn run_elt_arrays(arrs: Vec<ArrayRef>) -> Result<Arc<StringArray>> {
150-
let arr = elt(&arrs)?;
151-
let string_array = arr
152-
.as_any()
153-
.downcast_ref::<StringArray>()
154-
.ok_or_else(|| DataFusionError::Internal("expected Utf8".into()))?;
155-
Ok(Arc::new(string_array.clone()))
149+
fn run_elt_arrays(arrs: Vec<ArrayRef>) -> Result<StringArray> {
150+
run_elt_arrays_with(arrs, false)
151+
}
152+
153+
fn run_elt_arrays_with(arrs: Vec<ArrayRef>, ansi: bool) -> Result<StringArray> {
154+
let arr = elt(&arrs, ansi)?;
155+
Ok(as_string_array(&arr)?.clone())
156156
}
157157

158158
#[test]

‎datafusion/sqllogictest/test_files/spark/string/elt.slt‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,3 +59,143 @@ query T
5959
SELECT elt(1, 10, null)
6060
----
6161
10
62+
63+
########################################
64+
# ANSI mode = false (default): invalid indices return NULL
65+
########################################
66+
67+
# Index 0 -> NULL (Spark returns NULL when ANSI is off)
68+
query T
69+
SELECT elt(0::int, 'a', 'b');
70+
----
71+
NULL
72+
73+
# Negative index -> NULL
74+
query T
75+
SELECT elt(-1::int, 'a', 'b');
76+
----
77+
NULL
78+
79+
# Index far beyond the input list -> NULL
80+
query T
81+
SELECT elt(100::int, 'a', 'b', 'c');
82+
----
83+
NULL
84+
85+
# NULL index -> NULL regardless of mode
86+
query T
87+
SELECT elt(NULL::int, 'a', 'b');
88+
----
89+
NULL
90+
91+
# NULL value at the selected index -> NULL
92+
query T
93+
SELECT elt(2::int, 'a', NULL);
94+
----
95+
NULL
96+
97+
# Three-argument list, pick middle element
98+
query T
99+
SELECT elt(2::int, 'scala', 'java', 'python');
100+
----
101+
java
102+
103+
# Three-argument list, pick last element
104+
query T
105+
SELECT elt(3::int, 'scala', 'java', 'python');
106+
----
107+
python
108+
109+
# Mixed types get cast to string (Spark returns string)
110+
query T
111+
SELECT elt(2::int, 1, 2, 3);
112+
----
113+
2
114+
115+
# Vectorized: mix of valid, out-of-range, and NULL indices in ANSI-off mode
116+
statement ok
117+
CREATE TABLE elt_rows(idx INT, a STRING, b STRING, c STRING) AS VALUES
118+
(1, 'a1', 'b1', 'c1'),
119+
(2, 'a2', 'b2', 'c2'),
120+
(3, 'a3', 'b3', 'c3'),
121+
(0, 'a4', 'b4', 'c4'),
122+
(-1, 'a5', 'b5', 'c5'),
123+
(4, 'a6', 'b6', 'c6'),
124+
(NULL, 'a7', 'b7', 'c7');
125+
126+
query T
127+
SELECT elt(idx, a, b, c) FROM elt_rows ORDER BY a;
128+
----
129+
a1
130+
b2
131+
c3
132+
NULL
133+
NULL
134+
NULL
135+
NULL
136+
137+
statement ok
138+
DROP TABLE elt_rows;
139+
140+
########################################
141+
# ANSI mode = true: invalid indices raise ArrayIndexOutOfBoundsException
142+
########################################
143+
144+
statement ok
145+
set datafusion.execution.enable_ansi_mode = true;
146+
147+
# Valid indices still work
148+
query T
149+
SELECT elt(1::int, 'scala', 'java');
150+
----
151+
scala
152+
153+
query T
154+
SELECT elt(2::int, 'scala', 'java');
155+
----
156+
java
157+
158+
# NULL index still returns NULL (matches Spark: no error when index itself is NULL)
159+
query T
160+
SELECT elt(NULL::int, 'a', 'b');
161+
----
162+
NULL
163+
164+
# NULL value at valid index still returns NULL (only invalid indices error)
165+
query T
166+
SELECT elt(1::int, NULL, 'b');
167+
----
168+
NULL
169+
170+
# Out-of-range positive index errors
171+
statement error DataFusion error: Execution error: The index 3 is out of bounds\. The array has 2 elements\.
172+
SELECT elt(3::int, 'scala', 'java');
173+
174+
# Zero index errors
175+
statement error DataFusion error: Execution error: The index 0 is out of bounds\. The array has 2 elements\.
176+
SELECT elt(0::int, 'scala', 'java');
177+
178+
# Negative index errors
179+
statement error DataFusion error: Execution error: The index -1 is out of bounds\. The array has 2 elements\.
180+
SELECT elt(-1::int, 'scala', 'java');
181+
182+
# Large positive index errors
183+
statement error DataFusion error: Execution error: The index 100 is out of bounds\. The array has 3 elements\.
184+
SELECT elt(100::int, 'a', 'b', 'c');
185+
186+
# Vectorized: a batch that contains any invalid index errors in ANSI mode
187+
statement ok
188+
CREATE TABLE elt_ansi(idx INT, a STRING, b STRING) AS VALUES
189+
(1, 'a1', 'b1'),
190+
(2, 'a2', 'b2'),
191+
(3, 'a3', 'b3');
192+
193+
statement error DataFusion error: Execution error: The index 3 is out of bounds\. The array has 2 elements\.
194+
SELECT elt(idx, a, b) FROM elt_ansi;
195+
196+
statement ok
197+
DROP TABLE elt_ansi;
198+
199+
# Reset ANSI mode
200+
statement ok
201+
set datafusion.execution.enable_ansi_mode = false;

0 commit comments

Comments
 (0)