Skip to content

Commit 87a8d55

Browse files
committed
[fix](ivm) Net canceled deletes in MIN/MAX extreme counts
When the old extreme is NULL, the insert-side extreme became the new extreme but its stored count ignored deletes emitted for the same value by the same join refresh. The inflated MIN_COUNT/MAX_COUNT could later keep a deleted extreme instead of falling back. Initialize the count from the net insert/delete contributions at the selected extreme, and keep the complete refresh fallback when the surviving count cannot be proved. Key changes: - Compute the old-NULL count as insert count minus matching delete count; fall back when a delete is closer to the extreme or cancels every inserted row. - Add the MIN/MAX beyond-insert comparison and cover the net-count expression in the FE processor tests. - Fix the BE merge fixture to expect the two merged minima. - Make both MIN/MAX count regression suites wait for a task id distinct from the previous refresh. Unit Test: - IvmAggMinProcessorTest, IvmAggMaxProcessorTest, IvmAggDeltaHandlerTest, IvmNormalizeUniqKeysMTMVTest - regression-test: mtmv_p0/ivm test_ivm_minmax_count, test_ivm_minmax_count_join
1 parent c5d12fa commit 87a8d55

10 files changed

Lines changed: 260 additions & 11 deletions

File tree

‎be/test/exprs/aggregate/agg_count_at_min_max_test.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ TEST(CountAtMinMaxTest, Merge) {
128128

129129
ColumnInt64 result;
130130
function->insert_result_into(lhs, result);
131-
EXPECT_EQ(5, result.get_element(0));
131+
EXPECT_EQ(2, result.get_element(0));
132132
function->destroy(lhs);
133133
function->destroy(rhs);
134134
}

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/agg/IvmAggExtremalProcessor.java‎

Lines changed: 43 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@
5959
*
6060
* <ul>
6161
* <li>An inserted value better than the old extreme becomes the new extreme.</li>
62+
* <li>A group whose old extreme is NULL starts from the insert-side extreme, and its
63+
* count is first reduced by deletes that cancel the same value (a join can emit
64+
* such a delete for a row that the same refresh inserted elsewhere).</li>
6265
* <li>Otherwise the old extreme survives while
6366
* {@code oldCount + equal inserts - equal deletes > 0}.</li>
6467
* <li>If the old extreme is exhausted and no inserted value is better, the new extreme
@@ -149,26 +152,48 @@ public void appendApplyExpressions(IvmAggTarget target, IvmAggApplyContext apply
149152
// as unknown, so the guard below is a plain boolean.
150153
Expression insertedPresent = new Not(new IsNull(deltaInsert));
151154
Expression deletedPresent = new Not(new IsNull(deltaDelete));
155+
Expression oldExtremeNull = new IsNull(oldExtreme);
156+
Expression oldExtremePresent = new Not(oldExtremeNull);
152157
Expression insertedBetter = new And(insertedPresent, insertedIsBetter(deltaInsert, oldExtreme));
153158
Expression insertedEqual = new And(insertedPresent, new EqualTo(deltaInsert, oldExtreme));
154159
Expression deletedEqual = new And(deletedPresent, new EqualTo(deltaDelete, oldExtreme));
155160
Expression survivingOldCount = new Subtract(
156161
new Add(oldCount, new If(insertedEqual, deltaInsertCount, zero)),
157162
new If(deletedEqual, deltaDeleteCount, zero));
158163

164+
// When the old extreme is NULL the new extreme comes from the insert side. Its count
165+
// is the insert-side count minus the deletes that cancel the same value. That
166+
// subtraction is only possible when the delete-side extreme is exactly that value,
167+
// or is farther from the extreme (so no delete touches it). A delete closer to the
168+
// extreme, or deletes that cancel every inserted row, leaves the count unprovable
169+
// and keeps the complete-refresh fallback.
170+
Expression deleteEqualsInsert = new And(ImmutableList.of(
171+
deletedPresent, insertedPresent, new EqualTo(deltaDelete, deltaInsert)));
172+
Expression insertExtremeNetCount = new Subtract(deltaInsertCount, deltaDeleteCount);
173+
Expression insertExtremeProvable = new And(insertedPresent, new Or(ImmutableList.of(
174+
new IsNull(deltaDelete),
175+
new And(deletedPresent,
176+
deletedExtremeIsBeyondInsert(deltaDelete, deltaInsert)),
177+
new And(deleteEqualsInsert, new GreaterThan(insertExtremeNetCount, zero)))));
178+
Expression oldNullGuard = new And(oldExtremeNull, new Or(ImmutableList.of(
179+
new EqualTo(newTotalCount, zero),
180+
insertExtremeProvable)));
181+
159182
// The old extreme survives when no delete is below it and its remaining count is
160183
// positive. An inserted value that beats the old extreme also makes the merge
161184
// derivable without a complete refresh.
185+
Expression oldPresentNoDeletes = new And(oldExtremePresent, new IsNull(deltaDelete));
186+
Expression oldPresentBoundary = new And(oldExtremePresent,
187+
new And(deletedPresent,
188+
new And(deletedExtremeIsAtLeastOld(deltaDelete, oldExtreme),
189+
new Or(ImmutableList.of(
190+
insertedBetter,
191+
new GreaterThan(survivingOldCount, zero))))));
162192
Expression guardCond = new Or(ImmutableList.of(
163193
new EqualTo(newTotalCount, zero),
164-
new IsNull(oldExtreme),
165-
new IsNull(deltaDelete),
166-
new And(
167-
new And(deletedPresent,
168-
deletedExtremeIsAtLeastOld(deltaDelete, oldExtreme)),
169-
new Or(ImmutableList.of(
170-
insertedBetter,
171-
new GreaterThan(survivingOldCount, zero))))));
194+
oldNullGuard,
195+
oldPresentNoDeletes,
196+
oldPresentBoundary));
172197
Expression guard = new AssertTrue(guardCond, new StringLiteral(fallbackMessage()));
173198

174199
Expression newExtremeRaw = new CaseWhen(
@@ -181,10 +206,12 @@ public void appendApplyExpressions(IvmAggTarget target, IvmAggApplyContext apply
181206
Expression newExtremeGuarded = new If(guard, newExtremeRaw,
182207
new NullLiteral(newExtremeRaw.getDataType()));
183208

209+
Expression insertExtremeCount = new If(
210+
deleteEqualsInsert, insertExtremeNetCount, deltaInsertCount);
184211
Expression newCountRaw = new CaseWhen(
185212
ImmutableList.of(
186213
new WhenClause(new EqualTo(newTotalCount, zero), zero),
187-
new WhenClause(new IsNull(oldExtreme), deltaInsertCount),
214+
new WhenClause(oldExtremeNull, insertExtremeCount),
188215
new WhenClause(insertedBetter, deltaInsertCount)),
189216
survivingOldCount);
190217
Expression newCountGuarded = new If(guard, newCountRaw,
@@ -224,6 +251,13 @@ private void resolveDeltaSlot(IvmAggTarget target, String slotName, Map<String,
224251
/** True when the deleted extreme does not fall below/above the old extreme. */
225252
protected abstract Expression deletedExtremeIsAtLeastOld(Expression deltaDelete, Slot oldExtreme);
226253

254+
/**
255+
* True when the delete-side extreme is farther from the aggregate extreme than the
256+
* insert-side extreme, so no deleted row can cancel the insert-side extreme.
257+
*/
258+
protected abstract Expression deletedExtremeIsBeyondInsert(
259+
Expression deltaDelete, Expression deltaInsert);
260+
227261
/** Assertion message used when the delta is not sufficient and complete refresh is required. */
228262
protected abstract String fallbackMessage();
229263

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/agg/IvmAggMaxProcessor.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.apache.doris.mtmv.ivm.IvmFailureClassifier;
2121
import org.apache.doris.nereids.trees.expressions.Expression;
2222
import org.apache.doris.nereids.trees.expressions.GreaterThan;
23+
import org.apache.doris.nereids.trees.expressions.LessThan;
2324
import org.apache.doris.nereids.trees.expressions.LessThanEqual;
2425
import org.apache.doris.nereids.trees.expressions.Slot;
2526
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
@@ -57,6 +58,11 @@ protected Expression deletedExtremeIsAtLeastOld(Expression deltaDelete, Slot old
5758
return new LessThanEqual(deltaDelete, oldExtreme);
5859
}
5960

61+
@Override
62+
protected Expression deletedExtremeIsBeyondInsert(Expression deltaDelete, Expression deltaInsert) {
63+
return new LessThan(deltaDelete, deltaInsert);
64+
}
65+
6066
@Override
6167
protected String fallbackMessage() {
6268
return IvmFailureClassifier.MIN_MAX_BOUNDARY_MSG_PREFIX

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/agg/IvmAggMinProcessor.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
import org.apache.doris.mtmv.ivm.IvmFailureClassifier;
2121
import org.apache.doris.nereids.trees.expressions.Expression;
22+
import org.apache.doris.nereids.trees.expressions.GreaterThan;
2223
import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
2324
import org.apache.doris.nereids.trees.expressions.LessThan;
2425
import org.apache.doris.nereids.trees.expressions.Slot;
@@ -57,6 +58,11 @@ protected Expression deletedExtremeIsAtLeastOld(Expression deltaDelete, Slot old
5758
return new GreaterThanEqual(deltaDelete, oldExtreme);
5859
}
5960

61+
@Override
62+
protected Expression deletedExtremeIsBeyondInsert(Expression deltaDelete, Expression deltaInsert) {
63+
return new GreaterThan(deltaDelete, deltaInsert);
64+
}
65+
6066
@Override
6167
protected String fallbackMessage() {
6268
return IvmFailureClassifier.MIN_MAX_BOUNDARY_MSG_PREFIX

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/agg/IvmAggMaxProcessorTest.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,5 +69,8 @@ IvmAggStateKey.MAX_COUNT, slot(IvmUtil.ivmAggHiddenColumnName(3, "MAX_COUNT"),
6969
Assertions.assertTrue(visible.anyMatch(node -> node instanceof LessThanEqual));
7070
Assertions.assertTrue(visible.anyMatch(node -> node instanceof CaseWhen));
7171
Assertions.assertTrue(finalByName.containsKey(IvmUtil.ivmAggHiddenColumnName(3, "MAX_COUNT")));
72+
Expression hiddenCount = finalByName.get(IvmUtil.ivmAggHiddenColumnName(3, "MAX_COUNT"));
73+
Assertions.assertTrue(containsSlotSubtract(hiddenCount, "EXTREME_INS_COUNT", "EXTREME_DEL_COUNT"),
74+
"MAX count initialization must subtract matching deletes");
7275
}
7376
}

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/agg/IvmAggMinProcessorTest.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,5 +69,8 @@ IvmAggStateKey.MIN_COUNT, slot(IvmUtil.ivmAggHiddenColumnName(2, "MIN_COUNT"),
6969
Assertions.assertTrue(visible.anyMatch(node -> node instanceof GreaterThanEqual));
7070
Assertions.assertTrue(visible.anyMatch(node -> node instanceof CaseWhen));
7171
Assertions.assertTrue(finalByName.containsKey(IvmUtil.ivmAggHiddenColumnName(2, "MIN_COUNT")));
72+
Expression hiddenCount = finalByName.get(IvmUtil.ivmAggHiddenColumnName(2, "MIN_COUNT"));
73+
Assertions.assertTrue(containsSlotSubtract(hiddenCount, "EXTREME_INS_COUNT", "EXTREME_DEL_COUNT"),
74+
"MIN count initialization must subtract matching deletes");
7275
}
7376
}

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/agg/IvmAggProcessorTestBase.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.doris.nereids.trees.expressions.NamedExpression;
2222
import org.apache.doris.nereids.trees.expressions.Slot;
2323
import org.apache.doris.nereids.trees.expressions.SlotReference;
24+
import org.apache.doris.nereids.trees.expressions.Subtract;
2425
import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnion;
2526
import org.apache.doris.nereids.trees.expressions.functions.agg.Sum;
2627
import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapCount;
@@ -102,6 +103,28 @@ protected String slotName(Expression expression) {
102103
return ((Slot) expression).getName();
103104
}
104105

106+
/**
107+
* True when the expression subtracts the two delta count slots, which is what the
108+
* old-NULL branch must do to cancel deletes that match the insert-side extreme.
109+
*/
110+
protected boolean containsSlotSubtract(Expression expression, String leftName, String rightName) {
111+
if (expression instanceof Subtract) {
112+
Expression left = expression.child(0);
113+
Expression right = expression.child(1);
114+
if (left instanceof Slot && right instanceof Slot
115+
&& ((Slot) left).getName().contains(leftName)
116+
&& ((Slot) right).getName().contains(rightName)) {
117+
return true;
118+
}
119+
}
120+
for (Expression child : expression.children()) {
121+
if (containsSlotSubtract(child, leftName, rightName)) {
122+
return true;
123+
}
124+
}
125+
return false;
126+
}
127+
105128
protected void assertTwoBitmapDeltaOutputs(List<NamedExpression> outputs) {
106129
Assertions.assertEquals(2, outputs.size());
107130
Assertions.assertTrue(outputs.get(0).child(0) instanceof BitmapUnion);
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !initial --
3+
\N
4+
5+
-- !after_canceled_delete --
6+
1
7+
8+
-- !after_boundary --
9+
2
10+
11+
-- !source --
12+
2
13+

‎regression-test/suites/mtmv_p0/ivm/test_ivm_minmax_count.groovy‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,25 +65,38 @@ suite("test_ivm_minmax_count") {
6565
"unexpected IVM fallback: ${reason}")
6666
}
6767

68+
// A task that just finished is briefly absent from tasks(): the job removes it from its
69+
// running list before the MV history publishes it, so the newest row can temporarily be
70+
// the previous refresh. Require a different TaskId before trusting a terminal row.
71+
String lastTaskId = null
6872
def latestTask = {
6973
def taskResult
7074
Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({
7175
taskResult = sql_return_maparray("""
72-
SELECT Status, RefreshMode, IvmFallbackReason, ErrorMsg
76+
SELECT TaskId, Status, RefreshMode, IvmFallbackReason, ErrorMsg
7377
FROM tasks('type'='mv')
7478
WHERE MvDatabaseName = '${context.dbName}'
7579
AND MvName = 'ivm_mm_count_mv'
7680
ORDER BY CreateTime DESC, TaskId DESC LIMIT 1
7781
""")
7882
return !taskResult.isEmpty()
83+
&& taskResult[0].TaskId.toString() != lastTaskId
7984
&& taskResult[0].Status.toString() != 'PENDING'
8085
&& taskResult[0].Status.toString() != 'RUNNING'
8186
})
87+
lastTaskId = taskResult[0].TaskId.toString()
8288
return taskResult[0]
8389
}
8490

8591
sql """REFRESH MATERIALIZED VIEW ivm_mm_count_mv INCREMENTAL"""
8692
waitingMTMVTaskFinishedByMvName("ivm_mm_count_mv")
93+
lastTaskId = sql_return_maparray("""
94+
SELECT TaskId
95+
FROM tasks('type'='mv')
96+
WHERE MvDatabaseName = '${context.dbName}'
97+
AND MvName = 'ivm_mm_count_mv'
98+
ORDER BY CreateTime DESC, TaskId DESC LIMIT 1
99+
""")[0].TaskId.toString()
87100
order_qt_initial """
88101
SELECT category, min_score, max_score, row_count
89102
FROM ivm_mm_count_mv

0 commit comments

Comments
 (0)