Skip to content

Commit 76da858

Browse files
LiaCastanedagstvgrluvaton
authored
Add support for lambda column capture (apache#21323) (#130)
Part of apache#21172 Capture support wasn't implemented in the core lambda support to reduce PR size and because it requires further discussions not tied to basic support Lambda capture list_values_row_number helper to adjust a list to the lambda scope Make apache#18329 lambda-aware sqllogictests for lambda capture and CaseWhen unit tests for list_values_row_number This add breaking changes to unreleased items only --------- (cherry picked from commit 3b634aa) Co-authored-by: gstvg <[email protected]> Co-authored-by: Raz Luvaton <[email protected]>
1 parent 6ef2a10 commit 76da858

12 files changed

Lines changed: 674 additions & 86 deletions

File tree

‎datafusion/common/src/utils/mod.rs‎

Lines changed: 155 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,18 +30,22 @@ use arrow::array::{
3030
cast::AsArray,
3131
};
3232
use arrow::array::{
33-
Datum, GenericListArray, Int32Array, Int64Array, MutableArrayData, make_array,
33+
ArrowPrimitiveType, Datum, GenericListArray, Int32Array, Int64Array,
34+
MutableArrayData, PrimitiveArray, make_array,
3435
};
3536
use arrow::buffer::OffsetBuffer;
3637
use arrow::compute::kernels::cmp::neq;
3738
use arrow::compute::kernels::length::length;
3839
use arrow::compute::{SortColumn, SortOptions, partition};
39-
use arrow::datatypes::{DataType, Field, SchemaRef};
40+
use arrow::datatypes::{
41+
ArrowNativeType, DataType, Field, Int32Type, Int64Type, SchemaRef,
42+
};
4043
#[cfg(feature = "sql")]
4144
use sqlparser::{ast::Ident, dialect::GenericDialect, parser::Parser};
4245
use std::borrow::{Borrow, Cow};
4346
use std::cmp::{Ordering, min};
4447
use std::collections::HashSet;
48+
use std::iter::repeat_n;
4549
use std::num::NonZero;
4650
use std::ops::Range;
4751
use std::sync::{Arc, LazyLock};
@@ -1099,6 +1103,74 @@ fn truncate_list_nulls<O: OffsetSizeTrait>(
10991103
Ok(list.clone())
11001104
}
11011105

1106+
/// If `array` is a list or a map, returns a new array of the same length as it's inner values
1107+
/// where each value is the 1-based index of the sublist it's contained. Example:
1108+
///
1109+
/// `[[1], [2, 3], [4, 5, 6]] => [1, 2, 2, 3, 3, 3]`
1110+
///
1111+
/// Otherwise returns an error
1112+
pub fn list_values_row_number(array: &dyn Array) -> Result<ArrayRef> {
1113+
match array.data_type() {
1114+
DataType::List(_) => Ok(Arc::new(variable_size_list_values_row_number::<
1115+
Int32Type,
1116+
>(array.as_list().offsets()))),
1117+
DataType::LargeList(_) => Ok(Arc::new(variable_size_list_values_row_number::<
1118+
Int64Type,
1119+
>(array.as_list().offsets()))),
1120+
DataType::ListView(_) => Ok(Arc::new(variable_size_list_values_row_number::<
1121+
Int32Type,
1122+
>(array.as_list_view().offsets()))),
1123+
DataType::LargeListView(_) => {
1124+
Ok(Arc::new(variable_size_list_values_row_number::<Int64Type>(
1125+
array.as_list_view().offsets(),
1126+
)))
1127+
}
1128+
DataType::FixedSizeList(_, _) => {
1129+
let fixed_size_list = array.as_fixed_size_list();
1130+
1131+
Ok(Arc::new(fsl_values_row_number(
1132+
fixed_size_list.value_length(),
1133+
fixed_size_list.len(),
1134+
)?))
1135+
}
1136+
DataType::Map(_, _) => Ok(Arc::new(variable_size_list_values_row_number::<
1137+
Int32Type,
1138+
>(array.as_map().offsets()))),
1139+
other => _exec_err!("expected list, got {other}"),
1140+
}
1141+
}
1142+
1143+
/// [0, 2, 2, 5, 6] -> [0, 0, 2, 2, 2, 3]
1144+
fn variable_size_list_values_row_number<T: ArrowPrimitiveType>(
1145+
offsets: &[T::Native],
1146+
) -> PrimitiveArray<T> {
1147+
let mut rows_number = Vec::with_capacity(
1148+
offsets[offsets.len() - 1].to_usize().unwrap() - offsets[0].to_usize().unwrap(),
1149+
);
1150+
1151+
for (i, w) in offsets.windows(2).enumerate() {
1152+
let len = w[1].as_usize() - w[0].as_usize();
1153+
rows_number.extend(repeat_n(T::Native::usize_as(i), len));
1154+
}
1155+
1156+
PrimitiveArray::new(rows_number.into(), None)
1157+
}
1158+
1159+
/// (2, 3) -> [0, 0, 1, 1, 2, 2]
1160+
fn fsl_values_row_number(list_size: i32, array_len: usize) -> Result<Int32Array> {
1161+
let list_size = list_size.to_usize().ok_or_else(|| {
1162+
_exec_datafusion_err!("fsl_values_index: invalid list_size {list_size}")
1163+
})?;
1164+
1165+
let mut rows_number = Vec::with_capacity(list_size * array_len);
1166+
1167+
for i in 0..array_len {
1168+
rows_number.extend(repeat_n(i as i32, list_size));
1169+
}
1170+
1171+
Ok(PrimitiveArray::new(rows_number.into(), None))
1172+
}
1173+
11021174
#[cfg(test)]
11031175
mod tests {
11041176
use std::sync::Arc;
@@ -1535,4 +1607,85 @@ mod tests {
15351607
assert_eq!(res.values(), expected.values());
15361608
assert_eq!(res.offsets(), expected.offsets());
15371609
}
1610+
1611+
#[test]
1612+
fn test_list_array_values_row_number() {
1613+
assert_eq!(
1614+
variable_size_list_values_row_number::<Int32Type>(
1615+
&OffsetBuffer::from_lengths([1, 3, 0, 2,])
1616+
),
1617+
Int32Array::from(vec![0, 1, 1, 1, 3, 3])
1618+
);
1619+
1620+
assert_eq!(
1621+
variable_size_list_values_row_number::<Int32Type>(
1622+
&OffsetBuffer::from_lengths([])
1623+
),
1624+
Int32Array::new_null(0)
1625+
);
1626+
1627+
assert_eq!(
1628+
variable_size_list_values_row_number::<Int32Type>(
1629+
&OffsetBuffer::from_lengths([0])
1630+
),
1631+
Int32Array::new_null(0)
1632+
);
1633+
1634+
assert_eq!(
1635+
variable_size_list_values_row_number::<Int32Type>(
1636+
&OffsetBuffer::from_lengths([0, 0])
1637+
),
1638+
Int32Array::new_null(0)
1639+
);
1640+
1641+
assert_eq!(
1642+
variable_size_list_values_row_number::<Int32Type>(
1643+
&OffsetBuffer::from_lengths([1])
1644+
),
1645+
Int32Array::from(vec![0])
1646+
);
1647+
1648+
assert_eq!(
1649+
variable_size_list_values_row_number::<Int32Type>(
1650+
&OffsetBuffer::from_lengths([2])
1651+
),
1652+
Int32Array::from(vec![0, 0])
1653+
);
1654+
}
1655+
1656+
#[test]
1657+
fn test_fsl_values_row_number() {
1658+
assert_eq!(
1659+
fsl_values_row_number(2, 3).unwrap(),
1660+
Int32Array::from(vec![0, 0, 1, 1, 2, 2])
1661+
);
1662+
1663+
assert_eq!(
1664+
fsl_values_row_number(1, 3).unwrap(),
1665+
Int32Array::from(vec![0, 1, 2])
1666+
);
1667+
1668+
assert_eq!(
1669+
fsl_values_row_number(2, 1).unwrap(),
1670+
Int32Array::from(vec![0, 0])
1671+
);
1672+
1673+
assert_eq!(
1674+
fsl_values_row_number(2, 0).unwrap(),
1675+
Int32Array::new_null(0),
1676+
);
1677+
1678+
assert_eq!(
1679+
fsl_values_row_number(0, 2).unwrap(),
1680+
Int32Array::new_null(0),
1681+
);
1682+
1683+
assert_eq!(
1684+
fsl_values_row_number(0, 0).unwrap(),
1685+
Int32Array::new_null(0),
1686+
);
1687+
1688+
fsl_values_row_number(-1, 2).unwrap_err();
1689+
fsl_values_row_number(-1, 0).unwrap_err();
1690+
}
15381691
}

‎datafusion/expr/src/execution_props.rs‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
use crate::var_provider::{VarProvider, VarType};
1919
use chrono::{DateTime, Utc};
2020
use datafusion_common::HashMap;
21+
use datafusion_common::TableReference;
2122
use datafusion_common::alias::AliasGenerator;
2223
use datafusion_common::config::ConfigOptions;
2324
use std::sync::Arc;
@@ -42,6 +43,10 @@ pub struct ExecutionProps {
4243
pub config_options: Option<Arc<ConfigOptions>>,
4344
/// Providers for scalar variables
4445
pub var_providers: Option<HashMap<VarType, Arc<dyn VarProvider + Send + Sync>>>,
46+
/// Maps each lambda variable name to its lambda qualifier generated
47+
/// during physical planning. Populated by the physical planner for
48+
/// each lambda before calling `create_physical_expr`.
49+
pub lambda_variable_qualifier: HashMap<String, TableReference>,
4550
}
4651

4752
impl Default for ExecutionProps {
@@ -58,6 +63,7 @@ impl ExecutionProps {
5863
alias_generator: Arc::new(AliasGenerator::new()),
5964
config_options: None,
6065
var_providers: None,
66+
lambda_variable_qualifier: HashMap::new(),
6167
}
6268
}
6369

@@ -117,6 +123,22 @@ impl ExecutionProps {
117123
pub fn config_options(&self) -> Option<&Arc<ConfigOptions>> {
118124
self.config_options.as_ref()
119125
}
126+
127+
/// Adds a mapping for each variable to the given qualifier. Existing
128+
/// variables with conflicting names get's shadowed
129+
pub fn with_qualified_lambda_variables(
130+
mut self,
131+
qualifier: &TableReference,
132+
variables: &[String],
133+
) -> Self {
134+
for var in variables {
135+
self.lambda_variable_qualifier
136+
.entry_ref(var)
137+
.insert(qualifier.clone());
138+
}
139+
140+
self
141+
}
120142
}
121143

122144
#[cfg(test)]
@@ -126,7 +148,7 @@ mod test {
126148
fn debug() {
127149
let props = ExecutionProps::new();
128150
assert_eq!(
129-
"ExecutionProps { query_execution_start_time: None, alias_generator: AliasGenerator { next_id: 1 }, config_options: None, var_providers: None }",
151+
"ExecutionProps { query_execution_start_time: None, alias_generator: AliasGenerator { next_id: 1 }, config_options: None, var_providers: None, lambda_variable_qualifier: {} }",
130152
format!("{props:?}")
131153
);
132154
}

0 commit comments

Comments
 (0)