Skip to content

Commit 9e680c4

Browse files
committed
feat: fall back collect_list/collect_set on Spark 4.2 RESPECT NULLS
Spark 4.2 adds an ignoreNulls field to CollectList and CollectSet, and collect_list(x) RESPECT NULLS sets it to false, keeping null elements. The native path delegates to SparkCollectList/SparkCollectSet, which always drop nulls, so it would silently return a different result from Spark. Add a per-version CometCollectShim that reads ignoreNulls (always true on Spark 3.4 through 4.1, where the field is absent) and fall back to Spark in getSupportLevel when it is false. Also rename QueryPlanSerde.hasIncompatibleBufferAgg to hasNativeArrayBufferAgg to describe what it detects: an aggregate whose native ArrayType state cannot round-trip Spark's declared BinaryType buffer.
1 parent 7496e69 commit 9e680c4

10 files changed

Lines changed: 188 additions & 11 deletions

File tree

‎docs/source/contributor-guide/expression-audits/agg_funcs.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
- Spark 4.0.1 (audited 2026-06-24): only structural change is adding `with UnaryLike[Expression]` to the case class (no behavior change).
4747
- Spark 4.1.1 (audited 2026-06-24): identical to 4.0.1.
4848
- Comet implementation: native side delegates to `datafusion_spark::function::aggregate::collect::SparkCollectList`, which wraps `ArrayAggAccumulator` with `ignore_nulls = true` and converts a final NULL accumulator state to an empty array (matching Spark's `defaultResult`). The native return type is `List(Field, containsNull = true)`, while Spark uses `containsNull = false`. Because nulls are filtered before insertion, no nulls actually appear in the array, so this is a schema-shape difference only and tests using `checkSparkAnswerAndOperator` accept it (same pattern already in use for `collect_set`).
49+
- Spark 4.2 (preview): `CollectList` and `CollectSet` gain an `ignoreNulls` field (default `true`); `RESPECT NULLS` sets it to `false` and keeps null elements. The native path always drops nulls, so `CometCollectShim` reads the field per Spark version (always `true` on 3.4-4.1) and `CometCollectList` / `CometCollectSet` report `Unsupported` when it is `false`, falling back to Spark.
4950

5051
## median
5152

‎spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -866,7 +866,7 @@ case class CometExecRule(session: SparkSession)
866866
// Spark by tagging the feeding pure-Partial; the PartialMerge/Final stages then fall back
867867
// via the buffer-source check in doConvert.
868868
if (agg.aggregateExpressions.exists(_.mode == PartialMerge) &&
869-
QueryPlanSerde.hasIncompatibleBufferAgg(agg.aggregateExpressions)) {
869+
QueryPlanSerde.hasNativeArrayBufferAgg(agg.aggregateExpressions)) {
870870
findPartialAggInPlan(agg.child).foreach { partial =>
871871
if (canAggregateBeConverted(partial, Partial)) {
872872
partial.setTagValue(

‎spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -434,7 +434,7 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim {
434434
* are therefore only safe to run natively when every stage runs in Comet and there are at most
435435
* two stages (Partial + Final).
436436
*/
437-
def hasIncompatibleBufferAgg(aggExprs: Seq[AggregateExpression]): Boolean = {
437+
def hasNativeArrayBufferAgg(aggExprs: Seq[AggregateExpression]): Boolean = {
438438
aggExprs.exists(_.aggregateFunction match {
439439
case _: CollectList | _: CollectSet => true
440440
case _ => false

‎spark/src/main/scala/org/apache/comet/serde/aggregates.scala‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ import org.apache.spark.sql.types.{ByteType, DecimalType, DoubleType, IntegerTyp
2929
import org.apache.comet.CometConf.COMET_EXEC_STRICT_FLOATING_POINT
3030
import org.apache.comet.CometSparkSessionExtensions.{isSpark41Plus, withFallbackReason}
3131
import org.apache.comet.serde.QueryPlanSerde.{evalModeToProto, exprToProto, serializeDataType}
32-
import org.apache.comet.shims.CometEvalModeUtil
32+
import org.apache.comet.shims.{CometCollectShim, CometEvalModeUtil}
3333

3434
object CometMin extends CometAggregateExpressionSerde[Min] {
3535

@@ -784,13 +784,19 @@ object CometCollectSet extends CometAggregateExpressionSerde[CollectSet] {
784784
" `spark.comet.expression.CollectSet.allowIncompatible=true` is set.")
785785

786786
override def getSupportLevel(expr: CollectSet): SupportLevel = {
787-
SupportLevel
788-
.strictFloatingPointReason(
789-
expr.children.head.dataType,
790-
"collect_set on floating-point types " +
791-
"(Comet deduplicates NaN values while Spark treats each NaN as distinct)")
792-
.map(reason => Incompatible(Some(reason)))
793-
.getOrElse(Compatible())
787+
// The native path always drops null inputs. Spark 4.2 adds `RESPECT NULLS`
788+
// (`ignoreNulls = false`), which keeps nulls, so fall back there.
789+
if (!CometCollectShim.ignoreNulls(expr)) {
790+
Unsupported(Some("collect_set with RESPECT NULLS (ignoreNulls = false) is not supported"))
791+
} else {
792+
SupportLevel
793+
.strictFloatingPointReason(
794+
expr.children.head.dataType,
795+
"collect_set on floating-point types " +
796+
"(Comet deduplicates NaN values while Spark treats each NaN as distinct)")
797+
.map(reason => Incompatible(Some(reason)))
798+
.getOrElse(Compatible())
799+
}
794800
}
795801

796802
override def convert(
@@ -825,6 +831,16 @@ object CometCollectSet extends CometAggregateExpressionSerde[CollectSet] {
825831

826832
object CometCollectList extends CometAggregateExpressionSerde[CollectList] {
827833

834+
override def getSupportLevel(expr: CollectList): SupportLevel = {
835+
// The native path delegates to SparkCollectList, which always drops null inputs. Spark 4.2
836+
// adds `RESPECT NULLS` (`ignoreNulls = false`), which keeps nulls, so fall back there.
837+
if (!CometCollectShim.ignoreNulls(expr)) {
838+
Unsupported(Some("collect_list with RESPECT NULLS (ignoreNulls = false) is not supported"))
839+
} else {
840+
Compatible()
841+
}
842+
}
843+
828844
override def convert(
829845
aggExpr: AggregateExpression,
830846
expr: CollectList,

‎spark/src/main/scala/org/apache/spark/sql/comet/operators.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1554,7 +1554,7 @@ trait CometBaseAggregate {
15541554
// must fall back. The matching Partial is forced to fall back too (see CometExecRule), so the
15551555
// whole multi-stage chain runs consistently in one engine. See issue #4724.
15561556
if (modes.contains(PartialMerge) &&
1557-
QueryPlanSerde.hasIncompatibleBufferAgg(aggregate.aggregateExpressions) &&
1557+
QueryPlanSerde.hasNativeArrayBufferAgg(aggregate.aggregateExpressions) &&
15581558
findCometPartialAgg(aggregate.child).isEmpty) {
15591559
withFallbackReason(
15601560
aggregate,
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.comet.shims
21+
22+
import org.apache.spark.sql.catalyst.expressions.aggregate.{CollectList, CollectSet}
23+
24+
/**
25+
* Shim for the `ignoreNulls` flag on `CollectList` / `CollectSet`. Spark 3.4 through 4.1 have no
26+
* such field: these aggregates always drop null inputs, so this shim reports `true`. Spark 4.2
27+
* added `ignoreNulls` (settable to `false` via `RESPECT NULLS`), handled by the spark-4.2 shim.
28+
*/
29+
object CometCollectShim {
30+
def ignoreNulls(agg: CollectList): Boolean = true
31+
def ignoreNulls(agg: CollectSet): Boolean = true
32+
}
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.comet.shims
21+
22+
import org.apache.spark.sql.catalyst.expressions.aggregate.{CollectList, CollectSet}
23+
24+
/**
25+
* Shim for the `ignoreNulls` flag on `CollectList` / `CollectSet`. Spark 3.4 through 4.1 have no
26+
* such field: these aggregates always drop null inputs, so this shim reports `true`. Spark 4.2
27+
* added `ignoreNulls` (settable to `false` via `RESPECT NULLS`), handled by the spark-4.2 shim.
28+
*/
29+
object CometCollectShim {
30+
def ignoreNulls(agg: CollectList): Boolean = true
31+
def ignoreNulls(agg: CollectSet): Boolean = true
32+
}
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.comet.shims
21+
22+
import org.apache.spark.sql.catalyst.expressions.aggregate.{CollectList, CollectSet}
23+
24+
/**
25+
* Shim for the `ignoreNulls` flag on `CollectList` / `CollectSet`. Spark 3.4 through 4.1 have no
26+
* such field: these aggregates always drop null inputs, so this shim reports `true`. Spark 4.2
27+
* added `ignoreNulls` (settable to `false` via `RESPECT NULLS`), handled by the spark-4.2 shim.
28+
*/
29+
object CometCollectShim {
30+
def ignoreNulls(agg: CollectList): Boolean = true
31+
def ignoreNulls(agg: CollectSet): Boolean = true
32+
}
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.comet.shims
21+
22+
import org.apache.spark.sql.catalyst.expressions.aggregate.{CollectList, CollectSet}
23+
24+
/**
25+
* Shim for the `ignoreNulls` flag on `CollectList` / `CollectSet`. Spark 3.4 through 4.1 have no
26+
* such field: these aggregates always drop null inputs, so this shim reports `true`. Spark 4.2
27+
* added `ignoreNulls` (settable to `false` via `RESPECT NULLS`), handled by the spark-4.2 shim.
28+
*/
29+
object CometCollectShim {
30+
def ignoreNulls(agg: CollectList): Boolean = true
31+
def ignoreNulls(agg: CollectSet): Boolean = true
32+
}
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.comet.shims
21+
22+
import org.apache.spark.sql.catalyst.expressions.aggregate.{CollectList, CollectSet}
23+
24+
/**
25+
* Shim for the `ignoreNulls` flag on `CollectList` / `CollectSet`. Spark 4.2 added the field
26+
* (`ignoreNulls = false` via `RESPECT NULLS`), so this shim reports the actual value. The serde
27+
* falls back to Spark when it is `false`, since the native path always drops nulls.
28+
*/
29+
object CometCollectShim {
30+
def ignoreNulls(agg: CollectList): Boolean = agg.ignoreNulls
31+
def ignoreNulls(agg: CollectSet): Boolean = agg.ignoreNulls
32+
}

0 commit comments

Comments
 (0)