Skip to content

Commit 0ded66a

Browse files
authored
[feature](ivm) Support incremental refresh for array_agg and collect_list aggregates (#67575)
MVs containing `ARRAY_AGG` or `COLLECT_LIST` can now be refreshed incrementally (IVM): the visible array is the full aggregate state, so no hidden state column is needed. The delta aggregate emits insert-side and delete-side arrays and apply merges them as a multiset `except_all(concat(coalesce(old, []), ins), del)`, which keeps duplicate elements and NULLs correct. `COLLECT_LIST` skips NULL rows, so its polarity columns use the conditional-argument idiom. `ARRAY_AGG` keeps NULL elements, so NULL-filtering cannot build its polarity arrays; instead every change row is packed into a single `array_agg(struct(dml_factor, elem))` aggregate output and the delta top project derives the insert/delete polarity columns above the aggregate with `array_filter`/`array_map`, keeping the same two transient polarity slots for the multiset apply. `ARRAY_AGG` over JSONB/VARIANT elements (struct/map/array constructors all reject those types) is rejected during analysis with a precise reason, surfaced to the user at `CREATE MATERIALIZED VIEW` time. DISTINCT and the LIMIT variant of `collect_list` fall back to complete refresh. ## Key changes - add ARRAY_AGG and COLLECT_LIST aggregate kinds sharing one `IvmAggArrayProcessor` (two transient polarity delta slots, multiset apply, no hidden state); ARRAY_AGG derives its polarity columns above the delta aggregate through a new per-processor top-project hook - centralize polarity condition expressions and the typed empty array literal in `IvmAggExpressionBuilder` - regression suite `test_ivm_agg_array_1` covering grouped, scalar and mixed (array_agg + count + sum) MVs over insert/update/delete windows with NULL and non-NULL values Tracked in #65418
1 parent 8dd6aaf commit 0ded66a

13 files changed

Lines changed: 1078 additions & 4 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmAggDeltaHandler.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,17 @@ DeltaPlanParts buildDeltaSubPlan(LogicalAggregate<?> normalizedAgg,
228228
IvmUtil.buildRowIdHash(deltaAgg.getOutput().subList(0, groupKeySize)), Column.IVM_ROW_ID_COL);
229229
topOutputs.add(rowIdAlias);
230230

231+
// Processors that pack every change row into one aggregate column (ARRAY_AGG packs
232+
// (dml_factor, elem) into a struct array) derive their polarity columns here, above the
233+
// aggregate: an aggregate output cannot reference a sibling aggregate output, but a column
234+
// of the top delta project can. Derived outputs keep the same transient names the apply
235+
// stage resolves below.
236+
Map<String, Slot> deltaAggOutputByName = indexSlotsByName(deltaAgg.getOutput());
237+
for (IvmAggTarget target : aggMeta.getAggTargets()) {
238+
aggFunctionRegistry.appendDeltaTopProjectOutputs(
239+
target, deltaAggOutputByName, topOutputs, aggExpressionBuilder);
240+
}
241+
231242
Set<String> zeroDefaultDeltaOutputNames = collectZeroDefaultDeltaOutputNames(aggMeta);
232243
for (Slot slot : deltaAgg.getOutput()) {
233244
if (zeroDefaultDeltaOutputNames.contains(slot.getName())) {
Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.mtmv.ivm.agg;
19+
20+
import org.apache.doris.mtmv.ivm.IvmException;
21+
import org.apache.doris.mtmv.ivm.IvmFailureReason;
22+
import org.apache.doris.nereids.trees.expressions.Alias;
23+
import org.apache.doris.nereids.trees.expressions.ArrayItemReference;
24+
import org.apache.doris.nereids.trees.expressions.Expression;
25+
import org.apache.doris.nereids.trees.expressions.GreaterThan;
26+
import org.apache.doris.nereids.trees.expressions.LessThan;
27+
import org.apache.doris.nereids.trees.expressions.NamedExpression;
28+
import org.apache.doris.nereids.trees.expressions.Slot;
29+
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
30+
import org.apache.doris.nereids.trees.expressions.functions.agg.ArrayAgg;
31+
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFilter;
32+
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMap;
33+
import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateStruct;
34+
import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt;
35+
import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda;
36+
import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
37+
import org.apache.doris.nereids.trees.expressions.literal.StructLiteral;
38+
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
39+
import org.apache.doris.nereids.types.DataType;
40+
41+
import com.google.common.collect.ImmutableList;
42+
43+
import java.util.List;
44+
import java.util.Map;
45+
46+
/**
47+
* Processor for ARRAY_AGG(expr).
48+
*
49+
* <p>The visible array keeps NULL elements, so polarity columns cannot use the NULL-filtering
50+
* conditional-argument idiom (rows of the other polarity would leak into the array as NULL
51+
* elements). Instead every change row is packed into one struct field pair and aggregated once:
52+
*
53+
* <pre>array_agg(struct(dml_factor, elem))</pre>
54+
*
55+
* <p>The insert/delete polarity columns are then derived above the delta aggregate (see
56+
* {@link #appendDeltaTopProjectOutputs}) by splitting the packed pair array on the sign of the
57+
* factor field and projecting the elem field, one scalar pass per side. Both derived columns keep
58+
* the transient names the apply stage resolves, so the multiset merge is unchanged.
59+
*/
60+
class IvmAggArrayAggProcessor extends IvmAggArrayProcessor {
61+
private static final String PAIRS_SLOT = "ARRAY_PAIRS";
62+
/**
63+
* Field name of {@code struct(dml_factor, elem)} carrying the dml factor. {@link CreateStruct}
64+
* names unnamed fields {@code StructLiteral.COL_PREFIX + (i + 1)}, so the first field is "col1";
65+
* keep the string composed from the shared prefix so a rename cannot silently desync the
66+
* {@code element_at(x, ...)} field lookups below.
67+
*/
68+
private static final String FACTOR_FIELD = StructLiteral.COL_PREFIX + "1";
69+
/**
70+
* Field name of {@code struct(dml_factor, elem)} carrying the packed element (second field).
71+
*/
72+
private static final String ELEM_FIELD = StructLiteral.COL_PREFIX + "2";
73+
74+
@Override
75+
public boolean supportsOriginalFunction(AggregateFunction function) {
76+
if (!(function instanceof ArrayAgg)) {
77+
return false;
78+
}
79+
// The packed delta aggregates struct(dml_factor, elem) per change row, and struct fields
80+
// cannot carry JSONB/VARIANT (CreateStruct rejects them). ARRAY_AGG over such element types
81+
// is not incrementally maintainable. Throw with the precise reason instead of returning
82+
// false: false would surface only the registry's generic "unsupported aggregate for IVM:
83+
// array_agg" error, which misleadingly blames the whole function while ARRAY_AGG itself is
84+
// supported. This check also runs while CREATE MATERIALIZED VIEW analyzes the query, so the
85+
// user sees the exact cause at create time.
86+
DataType elemType = function.child(0).getDataType();
87+
if (elemType.isJsonType() || elemType.isVariantType()) {
88+
throw new IvmException(IvmFailureReason.AGG_UNSUPPORTED,
89+
"IVM: ARRAY_AGG over JSONB/VARIANT element type " + elemType.toSql()
90+
+ " is not incrementally maintainable (the delta packs struct(dml_factor,"
91+
+ " elem) and struct fields cannot carry JSONB/VARIANT), create the MV"
92+
+ " with a COMPLETE refresh instead: " + function.toSql());
93+
}
94+
return true;
95+
}
96+
97+
@Override
98+
public IvmAggFunctionKind handledFunctionKind() {
99+
return IvmAggFunctionKind.ARRAY_AGG;
100+
}
101+
102+
@Override
103+
void appendDeltaAggregateOutputs(IvmAggTarget target, Slot dmlFactorSlot,
104+
List<NamedExpression> outputs, IvmAggExpressionBuilder ctx) {
105+
Expression elem = target.getExprArgs().get(0);
106+
outputs.add(new Alias(new ArrayAgg(new CreateStruct(dmlFactorSlot, elem)),
107+
ctx.transientDeltaColumnName(target, PAIRS_SLOT)));
108+
}
109+
110+
@Override
111+
void appendDeltaTopProjectOutputs(IvmAggTarget target, Map<String, Slot> deltaAggOutputByName,
112+
List<NamedExpression> topOutputs, IvmAggExpressionBuilder ctx) {
113+
String pairsColumnName = ctx.transientDeltaColumnName(target, PAIRS_SLOT);
114+
Slot pairs = deltaAggOutputByName.get(pairsColumnName);
115+
if (pairs == null) {
116+
throw new IvmException(IvmFailureReason.PLAN_REWRITE_FAILED,
117+
"IVM agg delta rewrite failed to resolve packed array pairs output: "
118+
+ pairsColumnName + " for target " + target);
119+
}
120+
topOutputs.add(new Alias(splitPolarityPairs(true, pairs),
121+
ctx.transientDeltaColumnName(target, polaritySlotName(true))));
122+
topOutputs.add(new Alias(splitPolarityPairs(false, pairs),
123+
ctx.transientDeltaColumnName(target, polaritySlotName(false))));
124+
}
125+
126+
/**
127+
* Splits the packed {@code array<struct<dml_factor, elem>>} into the elements of one polarity
128+
* side, mirroring a conditional aggregate without filtering NULL elements first:
129+
*
130+
* <pre>array_map(x -> element_at(x, 'col2'),
131+
* array_filter(x -> element_at(x, 'col1') > 0, pairs)) // insert side
132+
* array_filter(x -> element_at(x, 'col1') < 0, pairs)) // delete side</pre>
133+
*
134+
* <p>The factor field of a change row is never NULL, so every pair is classified exactly once
135+
* and rows with factor zero are dropped on both sides, as before.
136+
*/
137+
private Expression splitPolarityPairs(boolean insertSide, Slot pairs) {
138+
ArrayItemReference pair = new ArrayItemReference("x", pairs);
139+
Expression factor = new ElementAt(pair.toSlot(), new StringLiteral(FACTOR_FIELD));
140+
Expression condition = insertSide ? new GreaterThan(factor, new TinyIntLiteral((byte) 0))
141+
: new LessThan(factor, new TinyIntLiteral((byte) 0));
142+
Expression filtered = new ArrayFilter(new Lambda(
143+
ImmutableList.of("x"), condition, ImmutableList.of(pair)));
144+
ArrayItemReference kept = new ArrayItemReference("y", filtered);
145+
Expression value = new ElementAt(kept.toSlot(), new StringLiteral(ELEM_FIELD));
146+
return new ArrayMap(new Lambda(ImmutableList.of("y"), value, ImmutableList.of(kept)));
147+
}
148+
}
Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.mtmv.ivm.agg;
19+
20+
import org.apache.doris.mtmv.ivm.IvmException;
21+
import org.apache.doris.mtmv.ivm.IvmFailureReason;
22+
import org.apache.doris.nereids.trees.expressions.Expression;
23+
import org.apache.doris.nereids.trees.expressions.Slot;
24+
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayConcat;
25+
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayExceptAll;
26+
import org.apache.doris.nereids.trees.expressions.functions.scalar.Coalesce;
27+
28+
import java.util.Map;
29+
30+
/**
31+
* Shared delta/apply logic for array-valued aggregate functions (ARRAY_AGG / COLLECT_LIST).
32+
*
33+
* <p>The stored visible array is the full aggregate state, so no hidden state column is needed: empty
34+
* groups and fully-deleted groups are handled by the central group-count machinery. Each subclass
35+
* decides how the delta aggregate emits its polarity columns, one over insert rows and one over delete
36+
* rows: COLLECT_LIST emits two conditional aggregates (NULL-filtering is safe because it skips NULL
37+
* rows), while ARRAY_AGG keeps NULL elements and therefore packs every change row into a single
38+
* {@code struct(dml_factor, elem)} array aggregate, deriving the two polarity columns above the delta
39+
* aggregate with scalar array functions. Apply merges them into the new visible array as a multiset:
40+
*
41+
* <pre>new = except_all(concat(coalesce(old, []), coalesce(ins, [])), coalesce(del, []))</pre>
42+
*
43+
* <p>The merge is a multiset identity: element order is irrelevant and duplicate elements cancel by
44+
* occurrence, which keeps updates and repeated values correct. NULL handling is delegated to each
45+
* underlying aggregate (ARRAY_AGG keeps NULL elements; COLLECT_LIST skips NULL rows).
46+
*/
47+
abstract class IvmAggArrayProcessor extends IvmAggFunctionProcessor {
48+
private static final String INS_SLOT = "ARRAY_INS";
49+
private static final String DEL_SLOT = "ARRAY_DEL";
50+
51+
/** Transient output-name suffix of the insert/delete polarity column for one side. */
52+
String polaritySlotName(boolean insertSide) {
53+
return insertSide ? INS_SLOT : DEL_SLOT;
54+
}
55+
56+
@Override
57+
void mapApplyDeltaSlots(IvmAggTarget target, Map<String, Slot> outputByName,
58+
Map<IvmAggDeltaSlotRef, Slot> applyDeltaSlots, Slot deltaGroupCountSlot,
59+
IvmAggExpressionBuilder ctx) {
60+
super.mapApplyDeltaSlots(target, outputByName, applyDeltaSlots, deltaGroupCountSlot, ctx);
61+
resolveDeltaSlot(target, INS_SLOT, outputByName, applyDeltaSlots, ctx);
62+
resolveDeltaSlot(target, DEL_SLOT, outputByName, applyDeltaSlots, ctx);
63+
}
64+
65+
@Override
66+
public void appendApplyExpressions(IvmAggTarget target, IvmAggApplyContext applyContext) {
67+
IvmAggExpressionBuilder ctx = applyContext.expressions();
68+
Slot oldArray = applyContext.rawMvSlot(target.getVisibleSlot().getName());
69+
Expression empty = ctx.emptyArrayLiteral(target.getVisibleSlot().getDataType());
70+
// The old MV side is genuinely NULL for groups that appear only in the delta (new groups).
71+
// The ins/del transient columns can never be NULL -- every delta group holds at least one
72+
// change row, and array_agg/collect_list yield an empty array (not NULL) for zero kept
73+
// elements -- so their COALESCE is defensive only.
74+
Expression insArray = applyContext.deltaSlotValue(target, deltaSlotRef(target, INS_SLOT));
75+
Expression delArray = applyContext.deltaSlotValue(target, deltaSlotRef(target, DEL_SLOT));
76+
applyContext.putFinalExpression(target, target.getVisibleSlot().getName(),
77+
new ArrayExceptAll(
78+
new ArrayConcat(new Coalesce(oldArray, empty), new Coalesce(insArray, empty)),
79+
new Coalesce(delArray, empty)));
80+
}
81+
82+
private void resolveDeltaSlot(IvmAggTarget target, String slotName, Map<String, Slot> outputByName,
83+
Map<IvmAggDeltaSlotRef, Slot> applyDeltaSlots, IvmAggExpressionBuilder ctx) {
84+
String columnName = ctx.transientDeltaColumnName(target, slotName);
85+
Slot slot = outputByName.get(columnName);
86+
if (slot == null) {
87+
throw new IvmException(IvmFailureReason.PLAN_REWRITE_FAILED,
88+
"IVM agg delta rewrite failed to resolve delta output slot: "
89+
+ columnName + " for target " + target);
90+
}
91+
applyDeltaSlots.put(deltaSlotRef(target, slotName), slot);
92+
}
93+
94+
private IvmAggDeltaSlotRef deltaSlotRef(IvmAggTarget target, String slotName) {
95+
return new IvmAggDeltaSlotRef(target.getOrdinal(), slotName);
96+
}
97+
}
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.mtmv.ivm.agg;
19+
20+
import org.apache.doris.nereids.trees.expressions.Alias;
21+
import org.apache.doris.nereids.trees.expressions.Expression;
22+
import org.apache.doris.nereids.trees.expressions.NamedExpression;
23+
import org.apache.doris.nereids.trees.expressions.Slot;
24+
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
25+
import org.apache.doris.nereids.trees.expressions.functions.agg.CollectList;
26+
27+
import java.util.List;
28+
29+
/**
30+
* Processor for COLLECT_LIST(expr) with exactly one argument.
31+
*
32+
* <p>COLLECT_LIST skips NULL rows, so the polarity columns use the plain conditional-argument idiom:
33+
* rows of the other polarity become NULL and are skipped by the aggregate itself.
34+
*
35+
* <p>The two-argument LIMIT variant is not incrementally maintainable and is intentionally not
36+
* supported (falls back to complete refresh).
37+
*/
38+
class IvmAggCollectListProcessor extends IvmAggArrayProcessor {
39+
@Override
40+
public boolean supportsOriginalFunction(AggregateFunction function) {
41+
return function instanceof CollectList && function.children().size() == 1;
42+
}
43+
44+
@Override
45+
public IvmAggFunctionKind handledFunctionKind() {
46+
return IvmAggFunctionKind.COLLECT_LIST;
47+
}
48+
49+
@Override
50+
void appendDeltaAggregateOutputs(IvmAggTarget target, Slot dmlFactorSlot,
51+
List<NamedExpression> outputs, IvmAggExpressionBuilder ctx) {
52+
Expression arg = target.getExprArgs().get(0);
53+
outputs.add(new Alias(buildPolarityAggregate(true, arg, dmlFactorSlot, ctx),
54+
ctx.transientDeltaColumnName(target, polaritySlotName(true))));
55+
outputs.add(new Alias(buildPolarityAggregate(false, arg, dmlFactorSlot, ctx),
56+
ctx.transientDeltaColumnName(target, polaritySlotName(false))));
57+
}
58+
59+
private AggregateFunction buildPolarityAggregate(boolean insertSide, Expression elem, Slot dmlFactorSlot,
60+
IvmAggExpressionBuilder ctx) {
61+
Expression filtered = insertSide
62+
? ctx.insertOnlyValue(elem, dmlFactorSlot)
63+
: ctx.deleteOnlyValue(elem, dmlFactorSlot);
64+
return new CollectList(filtered);
65+
}
66+
}

0 commit comments

Comments
 (0)