Skip to content

Commit f5cfc51

Browse files
committed
address review: restore sliced-input flatten test, drop liftFallbackReasons for stack
1 parent 164e51c commit f5cfc51

2 files changed

Lines changed: 31 additions & 13 deletions

File tree

‎native/operators/src/explode.rs‎

Lines changed: 30 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1880,18 +1880,39 @@ mod tests {
18801880

18811881
#[test]
18821882
fn flatten_struct_cols_nulls_fields_under_a_null_struct() {
1883-
// Row 1's struct is null but its field holds a value, which Arrow allows. Spark's
1884-
// `inline` turns a null struct into a row of nulls, so the value must not surface.
1885-
let field = Arc::new(Field::new("a", DataType::Int32, true));
1883+
// Row 1's struct is null but its fields hold values, which Arrow allows. Spark's `inline`
1884+
// turns a null struct into a row of nulls, so none of those values may surface.
1885+
let a: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), Some(2), None, Some(4)]));
1886+
let b: ArrayRef = Arc::new(Int32Array::from(vec![10, 20, 30, 40]));
1887+
let fields = vec![
1888+
Arc::new(Field::new("a", DataType::Int32, true)),
1889+
Arc::new(Field::new("b", DataType::Int32, false)),
1890+
];
18861891
let structs: ArrayRef = Arc::new(StructArray::new(
1887-
vec![Arc::clone(&field)].into(),
1888-
vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
1889-
Some(NullBuffer::from(vec![true, false, true])),
1892+
fields.clone().into(),
1893+
vec![a, b],
1894+
Some(NullBuffer::from(vec![true, false, true, true])),
18901895
));
1891-
let schema = Arc::new(Schema::new(vec![field]));
1896+
let schema = Arc::new(Schema::new(vec![
1897+
Field::new("a", DataType::Int32, true),
1898+
Field::new("b", DataType::Int32, true),
1899+
]));
1900+
let indices = HashSet::from([0]);
1901+
1902+
let batch = flatten_struct_cols(&[Arc::clone(&structs)], &schema, &indices).unwrap();
1903+
let a = batch.column(0).as_primitive::<Int32Type>();
1904+
let b = batch.column(1).as_primitive::<Int32Type>();
1905+
assert_eq!(a.iter().collect::<Vec<_>>(), [Some(1), None, None, Some(4)]);
1906+
assert_eq!(
1907+
b.iter().collect::<Vec<_>>(),
1908+
[Some(10), None, Some(30), Some(40)]
1909+
);
18921910

1893-
let batch = flatten_struct_cols(&[structs], &schema, &HashSet::from([0])).unwrap();
1911+
// A slice keeps the struct's mask aligned with its fields.
1912+
let batch = flatten_struct_cols(&[structs.slice(1, 2)], &schema, &indices).unwrap();
18941913
let a = batch.column(0).as_primitive::<Int32Type>();
1895-
assert_eq!(a.iter().collect::<Vec<_>>(), [Some(1), None, Some(3)]);
1914+
let b = batch.column(1).as_primitive::<Int32Type>();
1915+
assert_eq!(a.iter().collect::<Vec<_>>(), [None, None]);
1916+
assert_eq!(b.iter().collect::<Vec<_>>(), [None, Some(30)]);
18961917
}
18971918
}

‎spark/src/main/scala/org/apache/spark/sql/comet/operators.scala‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ import org.apache.comet.rules.CometExecRule
6262
import org.apache.comet.serde.{CometCollectBuffer, CometOperatorSerde, Compatible, OperatorOuterClass, QueryContextInterner, SupportLevel, Unsupported}
6363
import org.apache.comet.serde.OperatorOuterClass.{AggregateMode => CometAggregateMode, Operator}
6464
import org.apache.comet.serde.QueryPlanSerde
65-
import org.apache.comet.serde.QueryPlanSerde.{aggExprToProto, exprToProto, hasNonDefaultStringCollation, isStringCollationType, liftFallbackReasons, supportedSortType}
65+
import org.apache.comet.serde.QueryPlanSerde.{aggExprToProto, exprToProto, hasNonDefaultStringCollation, isStringCollationType, supportedSortType}
6666
import org.apache.comet.serde.operator.CometSink
6767
import org.apache.comet.shims.MergeRowsMetricsShim
6868

@@ -1952,9 +1952,6 @@ object CometExplodeExec extends CometOperatorSerde[GenerateExec] {
19521952
val childExprProto = exprToProto(childExpr, op.child.output)
19531953

19541954
if (childExprProto.isEmpty) {
1955-
// `stack` builds its array of structs here rather than taking it from the plan, so carry any
1956-
// reason recorded on the built nodes to the generator, where the operator collects it.
1957-
liftFallbackReasons(childExpr, op.generator)
19581955
return None
19591956
}
19601957

0 commit comments

Comments
 (0)