Skip to content

Commit 4f3abce

Browse files
authored
[fix](ivm) Resolve IVM identity keys by slot identity and materialize unprojected keys (#67669)
Two follow-ups from the #62606 review round, both around same-named identity keys of aggregate/full-keys MVs: 1. **Resolve IVM aggregate group keys by slot identity instead of name** — an aggregate delta over a join of two tables that both expose the same column name (`GROUP BY l.id, r.id`) resolved its group keys through name-keyed lookups. A name index collapses same-named slots onto the last match, so both keys were bound to one delta slot and a `(10, 20)` group was written as `(20, 20)`, diverging from COMPLETE. The delta-side group keys are now resolved and carried by slot identity (ordered with the aggregate metadata), the apply project emits each key output from its own delta slot, and the `ivm_use_full_keys` identity conjuncts resolve their delta side by identity first. The full-keys regression with a non-full-keys counterpart covers plain INSERT, full-keys, and MOW UPDATE refresh, each cross-checked against COMPLETE. 2. **Materialize unprojected full-keys identity keys at the project layer** — with `ivm_use_full_keys` every identity key must be materialized in the stored layout, either as a visible output or under a hidden key column. The result sink judged "already projected" by column name, so a same-named key from another table (selecting only `l.id` while grouping by `l.id, r.id`) was silently dropped and the key set lost one dimension. The sink now checks projection by slot identity, and materialization moves down to the first project that drops a key, so CREATE (result sink) and refresh (olap-table sink) plans produce the same hidden-key layout. A new regression asserts the hidden key column exists (DESC with hidden columns) and that INCREMENTAL matches COMPLETE. Tests: `IvmAggDeltaHandlerTest` (33) and `IvmNormalizeMTMVJoinTest` new cases pass; the new regression cases pass and the full ivm regression set (85 suites) is green on the materialization change. Trace issue: #65418
1 parent c1bff0d commit 4f3abce

10 files changed

Lines changed: 750 additions & 33 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmAggDeltaHandler.java‎

Lines changed: 69 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -110,15 +110,26 @@ static final class DeltaPlanParts {
110110
private final Slot deltaGroupCountSlot;
111111
/** Per-target delta slots consumed by aggregate function processors during apply. */
112112
private final Map<IvmAggDeltaSlotRef, Slot> applyDeltaSlots;
113-
/** Group key slots resolved from topDeltaProject output, keyed by column name. */
113+
/**
114+
* Group key slots resolved from the topDeltaProject output, ordered like
115+
* {@link IvmAggMeta#getGroupKeySlots()}. Same-named keys (e.g. l.id and r.id after
116+
* a join) must be addressed by position/identity through this list, never by name.
117+
*/
118+
private final List<Slot> deltaGroupKeySlots;
119+
/**
120+
* Name-keyed view of {@link #deltaGroupKeySlots}; with same-named group keys the map
121+
* keeps only the last slot, so it is only usable when all key names are distinct.
122+
*/
114123
private final Map<String, Slot> groupKeySlotsByName;
115124

116125
private DeltaPlanParts(LogicalProject<?> topDeltaProject, Slot rowIdSlot, Slot deltaGroupCountSlot,
117-
Map<IvmAggDeltaSlotRef, Slot> applyDeltaSlots, Map<String, Slot> groupKeySlotsByName) {
126+
Map<IvmAggDeltaSlotRef, Slot> applyDeltaSlots, List<Slot> deltaGroupKeySlots,
127+
Map<String, Slot> groupKeySlotsByName) {
118128
this.topDeltaProject = topDeltaProject;
119129
this.rowIdSlot = rowIdSlot;
120130
this.deltaGroupCountSlot = deltaGroupCountSlot;
121131
this.applyDeltaSlots = applyDeltaSlots;
132+
this.deltaGroupKeySlots = deltaGroupKeySlots;
122133
this.groupKeySlotsByName = groupKeySlotsByName;
123134
}
124135
}
@@ -237,19 +248,27 @@ DeltaPlanParts buildDeltaSubPlan(LogicalAggregate<?> normalizedAgg,
237248
aggFunctionRegistry.mapApplyDeltaSlots(
238249
target, outputByName, applyDeltaSlots, deltaGroupCountSlot, aggExpressionBuilder);
239250
}
240-
Map<String, Slot> groupKeySlotsByName = new LinkedHashMap<>();
251+
// Group keys are resolved by slot identity, not by name: same-named keys (e.g.
252+
// l.id / r.id after a join) collapse to the last matching slot under a name index.
253+
List<Slot> deltaOutputSlots = topDeltaProject.getOutput();
254+
List<Slot> deltaGroupKeySlots = new ArrayList<>(groupKeySize);
241255
for (Slot groupKey : aggMeta.getGroupKeySlots()) {
242-
Slot resolved = outputByName.get(groupKey.getName());
256+
Slot resolved = helper.findSlotByExprId(deltaOutputSlots, groupKey);
243257
if (resolved == null) {
244258
throw new IvmException(IvmFailureReason.PLAN_REWRITE_FAILED,
245259
"IVM agg delta rewrite failed to resolve delta group key slot: "
246-
+ groupKey.getName());
260+
+ groupKey);
247261
}
248-
groupKeySlotsByName.put(groupKey.getName(), resolved);
262+
deltaGroupKeySlots.add(resolved);
263+
}
264+
// Name-keyed view, kept only for lookups on distinct key names (see field comment).
265+
Map<String, Slot> groupKeySlotsByName = new LinkedHashMap<>();
266+
for (Slot resolved : deltaGroupKeySlots) {
267+
groupKeySlotsByName.put(resolved.getName(), resolved);
249268
}
250269

251270
return new DeltaPlanParts(topDeltaProject, outputByName.get(Column.IVM_ROW_ID_COL),
252-
deltaGroupCountSlot, applyDeltaSlots, groupKeySlotsByName);
271+
deltaGroupCountSlot, applyDeltaSlots, deltaGroupKeySlots, groupKeySlotsByName);
253272
}
254273

255274
/**
@@ -304,9 +323,8 @@ LogicalProject<?> buildApplyPlan(LogicalAggregate<?> normalizedAgg,
304323
"negative group count");
305324
finalByColumnName.put(Column.IVM_ROW_ID_COL, delta.rowIdSlot);
306325
finalByColumnName.put(aggMeta.getGroupCountSlot().getName(), newGroupCount);
307-
for (Slot groupKey : aggMeta.getGroupKeySlots()) {
308-
finalByColumnName.put(groupKey.getName(), deltaGroupKey(delta, groupKey.getName()));
309-
}
326+
// Group keys are emitted from the delta side below, resolved by slot identity, and
327+
// must not enter the name-keyed map: same-named keys would overwrite each other.
310328

311329
Set<String> visibleColumnNames = new HashSet<>();
312330
for (IvmAggTarget target : aggMeta.getAggTargets()) {
@@ -330,7 +348,13 @@ LogicalProject<?> buildApplyPlan(LogicalAggregate<?> normalizedAgg,
330348
// Keep the normalized aggregate schema here. The normalize-added top project computes row-id above this
331349
// project, and the final sink project reorders columns by MV schema.
332350
for (Slot target : normalizedAgg.getOutput()) {
333-
Expression expr = finalByColumnName.get(target.getName());
351+
// A group key output takes the delta-side key slot: a group that is new in the
352+
// delta has no MV row to copy from. Same-named keys are disambiguated by slot
353+
// identity against aggMeta's ordered group keys.
354+
int groupKeyIndex = groupKeyIndexIn(target, aggMeta);
355+
Expression expr = groupKeyIndex >= 0
356+
? delta.deltaGroupKeySlots.get(groupKeyIndex)
357+
: finalByColumnName.get(target.getName());
334358
if (expr == null) {
335359
throw new IvmException(IvmFailureReason.PLAN_REWRITE_FAILED,
336360
"IVM agg delta rewrite missing output expression for column: "
@@ -350,11 +374,18 @@ private ImmutableList<Expression> buildApplyJoinConjuncts(IvmIncrRefreshContext
350374
List<Slot> identityKeys = rewriteResult == null ? null : rewriteResult.getIdentityKeySlots();
351375
if (identityKeys != null) {
352376
for (Slot identityKey : identityKeys) {
377+
// The MV side is matched by column name (the MV key columns are the
378+
// normalized outputs). When an MV selects same-named group keys under
379+
// output aliases, the alias differs from the key slot name and this lookup
380+
// misses: the identity conjunct is skipped and the row-id conjunct still
381+
// guards the join, so no wrong value is produced. The delta side below is
382+
// always resolved by slot identity (findDeltaKeyByIdentity), so a matched
383+
// identity key can never be paired with the wrong delta slot.
353384
Slot mvKey = helper.findSlotByNameOrNull(rawMvScan.getOutput(), identityKey.getName());
354385
if (mvKey == null) {
355386
continue;
356387
}
357-
Slot deltaKey = findDeltaKeyByName(delta, identityKey.getName());
388+
Slot deltaKey = findDeltaKeyByIdentity(delta, identityKey);
358389
if (deltaKey == null) {
359390
continue;
360391
}
@@ -367,6 +398,21 @@ private ImmutableList<Expression> buildApplyJoinConjuncts(IvmIncrRefreshContext
367398
return conjuncts.build();
368399
}
369400

401+
/**
402+
* Resolves the delta-side key slot for an identity key by slot identity first (both
403+
* live in the same normalized plan, so same-named keys such as l.id / r.id match their
404+
* own delta slot), falling back to the name-keyed lookup for legacy callers whose keys
405+
* have distinct names.
406+
*/
407+
private Slot findDeltaKeyByIdentity(DeltaPlanParts delta, Slot identityKey) {
408+
for (Slot keySlot : delta.deltaGroupKeySlots) {
409+
if (keySlot.getExprId().equals(identityKey.getExprId())) {
410+
return keySlot;
411+
}
412+
}
413+
return findDeltaKeyByName(delta, identityKey.getName());
414+
}
415+
370416
private Slot findDeltaKeyByName(DeltaPlanParts delta, String name) {
371417
Slot direct = delta.groupKeySlotsByName.get(name);
372418
if (direct != null) {
@@ -449,15 +495,6 @@ private Expression deltaGroupCount(DeltaPlanParts delta) {
449495
return delta.deltaGroupCountSlot;
450496
}
451497

452-
private Expression deltaGroupKey(DeltaPlanParts delta, String name) {
453-
Slot slot = delta.groupKeySlotsByName.get(name);
454-
if (slot == null) {
455-
throw new IvmException(IvmFailureReason.PLAN_REWRITE_FAILED,
456-
"IVM agg delta rewrite failed to resolve delta group key: " + name);
457-
}
458-
return slot;
459-
}
460-
461498
/**
462499
* Collects delta output names where NULL should be normalized to zero before apply.
463500
*
@@ -487,4 +524,15 @@ private Map<String, Slot> indexSlotsByName(List<Slot> slots) {
487524
return slotByName;
488525
}
489526

527+
/** Index of {@code slot} among the aggregate group keys, or -1 when it is not a group key. */
528+
private int groupKeyIndexIn(Slot slot, IvmAggMeta aggMeta) {
529+
List<Slot> groupKeys = aggMeta.getGroupKeySlots();
530+
for (int i = 0; i < groupKeys.size(); i++) {
531+
if (groupKeys.get(i).getExprId().equals(slot.getExprId())) {
532+
return i;
533+
}
534+
}
535+
return -1;
536+
}
537+
490538
}

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmDeltaRewriteHelper.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,20 @@ Slot findSlotByNameOrNull(List<Slot> slots, String name) {
8181
return null;
8282
}
8383

84+
/**
85+
* Finds the slot in {@code slots} whose ExprId equals {@code target}'s, or null. Slots
86+
* that share a name (e.g. l.id / r.id after a join) must be resolved by slot identity
87+
* instead of a name-keyed lookup, which collapses same-named slots onto one of them.
88+
*/
89+
public Slot findSlotByExprId(List<Slot> slots, Slot target) {
90+
for (Slot slot : slots) {
91+
if (slot.getExprId().equals(target.getExprId())) {
92+
return slot;
93+
}
94+
}
95+
return null;
96+
}
97+
8498
/**
8599
* Adds the root-level fallback guard for a non-deterministic MV row-id. Inserts remain valid;
86100
* any delete delta fails so the caller can fall back to a full refresh.

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/IvmNormalizeMTMV.java‎

Lines changed: 35 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -383,9 +383,19 @@ public Plan visitLogicalProject(LogicalProject<? extends Plan> project, Normaliz
383383
NamedExpression projected = findProjectedKey(baseOutputs, keySlot);
384384
if (projected != null) {
385385
survivingKeys.add(projected.toSlot());
386-
} else {
387-
extendedOutputs.add(keySlot);
386+
} else if (IvmUtil.isIvmHiddenColumn(keySlot.getName())) {
387+
// Already materialized by a lower layer: keep the hidden key slot as is.
388388
survivingKeys.add(keySlot);
389+
} else {
390+
// An unprojected business key is materialized here as a hidden key column
391+
// instead of being passed through: the result sink (CREATE) and the
392+
// olap-table sink (refresh) then see the same hidden layout and agree on
393+
// the hidden column names. Keys are matched by slot identity, so a
394+
// same-named key from another table (e.g. r.id next to an output l.id)
395+
// still gets its own hidden column.
396+
Alias hiddenAlias = materializeHiddenKey(keySlot);
397+
extendedOutputs.add(hiddenAlias);
398+
survivingKeys.add(hiddenAlias.toSlot());
389399
}
390400
}
391401
finalOutputs = ImmutableList.copyOf(extendedOutputs);
@@ -416,6 +426,21 @@ private NamedExpression findProjectedKey(List<NamedExpression> outputs, Slot key
416426
return null;
417427
}
418428

429+
/**
430+
* Materializes an unprojected identity key under a unique hidden key column name. The
431+
* same helper is used by the project layer (where refresh and CREATE plans first drop a
432+
* key) and by the sink (direct-child fallback), so both paths agree on hidden names.
433+
*
434+
* <p>Note: the returned alias owns a fresh ExprId, so a later delta rewrite cannot match
435+
* the hidden column to the original key slot by identity; it resolves it through the
436+
* hidden column name (see the agg handler's sanitized-name fallback).
437+
*/
438+
private Alias materializeHiddenKey(Slot keySlot) {
439+
String hiddenName = Column.IVM_KEY_COL_PREFIX + (++sinkKeyCounter) + "_"
440+
+ IvmUtil.sanitizeIvmKeyName(keySlot.getName()) + "_COL__";
441+
return new Alias(keySlot, hiddenName);
442+
}
443+
419444
@Override
420445
public Plan visitLogicalFilter(LogicalFilter<? extends Plan> filter, NormalizeContext context) {
421446
Plan result = filter.withChildren(child -> child.accept(this, context.afterNonSink()));
@@ -927,19 +952,20 @@ public Plan visitLogicalResultSink(LogicalResultSink<? extends Plan> sink, Norma
927952
List<Slot> sinkKeys = new ArrayList<>();
928953
List<NamedExpression> finalOutputs = new ArrayList<>(baseOutputs);
929954
if (childKeys != null && !childKeys.isEmpty()) {
930-
Set<String> outputNames = finalOutputs.stream()
931-
.map(NamedExpression::getName)
932-
.collect(Collectors.toSet());
933955
for (Slot keySlot : childKeys) {
934956
String keyName = keySlot.getName();
935957
if (IvmUtil.isIvmHiddenColumn(keyName)) {
936958
sinkKeys.add(keySlot);
937-
} else if (outputNames.contains(keyName)) {
959+
} else if (findProjectedKey(finalOutputs, keySlot) != null) {
960+
// The key is already materialized by an output that directly emits the
961+
// key slot (bare slot or alias over it). Matching by slot identity -
962+
// as the project layer does - instead of by name is required: a
963+
// same-named key from another table (e.g. r.id next to an output l.id)
964+
// must still be materialized under its own hidden column, or the
965+
// full-keys identity set silently loses one dimension.
938966
sinkKeys.add(keySlot);
939967
} else {
940-
String hiddenName = Column.IVM_KEY_COL_PREFIX + (++sinkKeyCounter) + "_"
941-
+ IvmUtil.sanitizeIvmKeyName(keyName) + "_COL__";
942-
Alias hiddenAlias = new Alias(keySlot, hiddenName);
968+
Alias hiddenAlias = materializeHiddenKey(keySlot);
943969
finalOutputs.add(hiddenAlias);
944970
sinkKeys.add(hiddenAlias.toSlot());
945971
}

0 commit comments

Comments
 (0)