Skip to content

Commit 9c9ff57

Browse files
committed
Evaluate a masked argument through one copy, so shuffle keeps its state
A masked argument was evaluated through its rebuilt copy, which reads its columns from a batch of just those columns, only for a batch with a NULL in an earlier argument, and through the original for every other batch. Rebuilding an argument whose column indices change gives it new state, so array_union(a, shuffle(b)) restarted shuffle's random sequence each time evaluation moved between the two, and drew different permutations from Spark's. The rebuilt copy now evaluates a masked argument for every batch. A unit test evaluates array_union(a, shuffle(b)) over a batch without a NULL array, one with, and one without again, against the unwrapped function, which draws the same permutations as Spark when b is NULL wherever a is. A Spark 4.0+ SQL fixture reads 8200 rows from one file, so that only the scan's second batch has NULL arrays. Both fail with the previous evaluation.
1 parent ad2fe3a commit 9c9ff57

2 files changed

Lines changed: 88 additions & 6 deletions

File tree

‎native/spark-expr/src/conditional_funcs/null_short_circuit.rs‎

Lines changed: 52 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,9 @@ struct Arg {
243243
field: Option<FieldRef>,
244244
/// For an argument evaluated for only the rows where no earlier argument is NULL, the input
245245
/// columns it reads, and the argument reading them from a batch of just those columns, so
246-
/// that skipping rows filters no other column
246+
/// that skipping rows filters no other column. It is evaluated in place of `expr` for every
247+
/// batch, even one with no NULL, because rebuilding the argument gave it its own copy of any
248+
/// state kept across batches, such as the random generator of `shuffle`
247249
masked: Option<(Vec<usize>, Arc<dyn PhysicalExpr>)>,
248250
}
249251

@@ -264,12 +266,16 @@ impl Arg {
264266
/// Evaluates the argument, for just the rows outside `nulls` if it is masked, returning a
265267
/// value for every row of the batch.
266268
fn evaluate(&self, batch: &RecordBatch, nulls: Option<&NullBuffer>) -> Result<ColumnarValue> {
267-
match (&self.masked, nulls) {
268-
(Some((projection, projected)), Some(nulls)) => {
269+
let Some((projection, projected)) = &self.masked else {
270+
return self.expr.evaluate(batch);
271+
};
272+
let batch = batch.project(projection)?;
273+
match nulls {
274+
Some(nulls) => {
269275
let rows = BooleanArray::new(nulls.inner().clone(), None);
270-
projected.evaluate_selection(&batch.project(projection)?, &rows)
276+
projected.evaluate_selection(&batch, &rows)
271277
}
272-
_ => self.expr.evaluate(batch),
278+
None => projected.evaluate(&batch),
273279
}
274280
}
275281
}
@@ -305,13 +311,14 @@ fn project(expr: &Arc<dyn PhysicalExpr>) -> Result<(Vec<usize>, Arc<dyn Physical
305311
mod tests {
306312
use super::*;
307313
use crate::{
308-
create_query_context_map, Cast, EvalMode, ListExtract, SparkArrayPositionFunc,
314+
create_query_context_map, Cast, EvalMode, ListExtract, ShuffleExpr, SparkArrayPositionFunc,
309315
SparkArraySlice, SparkCastOptions,
310316
};
311317
use arrow::array::{ArrayRef, Int32Array, Int64Array, ListArray, StringArray};
312318
use arrow::datatypes::{Field, Int32Type};
313319
use datafusion::common::config::ConfigOptions;
314320
use datafusion::common::ScalarValue;
321+
use datafusion::functions_nested::set_ops::array_union_udf;
315322
use datafusion::logical_expr::ScalarUDF;
316323
use datafusion::physical_expr::expressions::{col, lit, CaseExpr, IsNotNullExpr};
317324
use datafusion::physical_expr::ScalarFunctionExpr;
@@ -599,6 +606,45 @@ mod tests {
599606
assert_eq!(actual.as_ref(), expected.as_ref());
600607
}
601608

609+
#[test]
610+
fn stateful_argument_keeps_its_state_across_batches() {
611+
let list = DataType::new_list(DataType::Int32, true);
612+
let schema = Arc::new(Schema::new(vec![
613+
Field::new("a", list.clone(), true),
614+
Field::new("b", list.clone(), true),
615+
]));
616+
// array_union(a, shuffle(b)). The masked shuffle reads `b` from the first column of a
617+
// batch of just that column, so it is rebuilt, with a generator of its own.
618+
let union = || {
619+
let shuffle = Arc::new(ShuffleExpr::new(col("b", &schema).unwrap(), 42));
620+
let udf = array_union_udf().as_ref().clone();
621+
function(udf, vec![col("a", &schema).unwrap(), shuffle], list.clone())
622+
};
623+
// An empty array and [1, 2, 3, 4] on each row that is not NULL. A shuffle draws nothing
624+
// for a NULL row, so the unwrapped function, which evaluates it for the rows whose array
625+
// is NULL as well, draws the same permutations as Spark.
626+
let batch = |valid: &[bool]| {
627+
let four = || (1..=4).map(Some).collect::<Vec<_>>();
628+
let a = valid.iter().map(|&v| v.then(Vec::<Option<i32>>::new));
629+
let b = valid.iter().map(|&v| v.then(four));
630+
let columns: Vec<ArrayRef> = vec![
631+
Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(a)),
632+
Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(b)),
633+
];
634+
RecordBatch::try_new(Arc::clone(&schema), columns).unwrap()
635+
};
636+
let unguarded = union();
637+
let guarded = NullShortCircuit::wrap(union(), &schema).unwrap();
638+
assert!(guarded.is::<NullShortCircuit>());
639+
// The permutations carry on from one batch to the next, whether or not it has a NULL
640+
for valid in [vec![true, true], vec![false, true, true], vec![true, true]] {
641+
let batch = batch(&valid);
642+
let expected = evaluate(&unguarded, &batch).unwrap();
643+
let actual = evaluate(&guarded, &batch).unwrap();
644+
assert_eq!(actual.as_ref(), expected.as_ref());
645+
}
646+
}
647+
602648
/// Evaluates `child`, recording the rows and columns of each batch that it is evaluated for.
603649
#[derive(Debug)]
604650
struct RowRecorder {
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
-- Licensed to the Apache Software Foundation (ASF) under one
2+
-- or more contributor license agreements. See the NOTICE file
3+
-- distributed with this work for additional information
4+
-- regarding copyright ownership. The ASF licenses this file
5+
-- to you under the Apache License, Version 2.0 (the
6+
-- "License"); you may not use this file except in compliance
7+
-- with the License. You may obtain a copy of the License at
8+
--
9+
-- http://www.apache.org/licenses/LICENSE-2.0
10+
--
11+
-- Unless required by applicable law or agreed to in writing,
12+
-- software distributed under the License is distributed on an
13+
-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
-- KIND, either express or implied. See the License for the
15+
-- specific language governing permissions and limitations
16+
-- under the License.
17+
18+
-- array_union evaluates shuffle(b) only for the rows whose array is not NULL, as in
19+
-- array_null_short_circuit.sql. shuffle keeps one random generator for every batch of a partition,
20+
-- so its permutations have to carry on from a batch with no NULL array to one with some. The two
21+
-- argument shuffle(array, seed) form only exists in Spark 4.0+.
22+
-- https://github.com/apache/datafusion-comet/issues/6613
23+
24+
-- MinSparkVersion: 4.0
25+
26+
statement
27+
CREATE TABLE array_null_short_circuit_shuffle(id bigint, a array<int>, b array<int>) USING parquet
28+
29+
-- A single file, which the scan reads in batches of 8192 rows. Only the second batch has NULL arrays.
30+
statement
31+
INSERT INTO array_null_short_circuit_shuffle
32+
SELECT id, IF(id >= 8192 AND id % 2 = 0, NULL, array(0)), array(1, 2, 3, 4)
33+
FROM range(0, 8200, 1, 1)
34+
35+
query expect_native(array_union, shuffle)
36+
SELECT id, array_union(a, shuffle(b, 42)) FROM array_null_short_circuit_shuffle

0 commit comments

Comments
 (0)