Skip to content

Commit e777b7b

Browse files
committed
[fix](ivm) Give every data type a non-null default value for hidden slots
An IVM base table can carry a DATETIME sequence column through function_column.sequence_col, which makes its hidden __DORIS_SEQUENCE_COL__ a DATETIMEV2 slot. The incremental rewrite asks for the hidden slot's default value when a join's snapshot side does not produce the column, but the default-value handling only covered numeric, string-like and bitmap types, so planning failed with "no default value for IVM hidden slot: __DORIS_SEQUENCE_COL__, type=DATETIMEV2(6)". Key changes: - Add an abstract DataType#defaultValue() and implement it in every concrete type with that type's non-null default value. - Resolve common hidden slot defaults through DataType#defaultValue() in IvmUtil instead of branching on type categories. - Add an IVM regression suite covering a DATETIME sequence column with aggregate and plain join materialized views. Unit Test: - IvmUtilTest#testCommonHiddenSlots - IvmUtilTest#testDefaultValueForEveryType - regression-test/suites/mtmv_p0/ivm/test_ivm_datetime_sequence.groovy
1 parent b9907e2 commit e777b7b

46 files changed

Lines changed: 615 additions & 19 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmUtil.java‎

Lines changed: 24 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
import org.apache.doris.nereids.trees.expressions.IsNull;
3636
import org.apache.doris.nereids.trees.expressions.NamedExpression;
3737
import org.apache.doris.nereids.trees.expressions.Slot;
38-
import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapEmpty;
3938
import org.apache.doris.nereids.trees.expressions.functions.scalar.MurmurHash3128;
4039
import org.apache.doris.nereids.trees.expressions.functions.scalar.Nvl;
4140
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
@@ -85,14 +84,22 @@ public static boolean isIvmHiddenColumn(String columnName) {
8584
}
8685

8786
/**
88-
* A base scan's own storage column that must not be carried into an IVM plan, because its name
89-
* collides with a column the delta rewrite binds by name and it is read by the plan above.
87+
* A base scan's own storage column that must not be carried into an IVM plan.
9088
* <p>
91-
* The base table's sequence column ({@code Column.SEQUENCE_COL}) is exactly such a column: IVM owns
92-
* the MV's sequence and binds it by name, so a base sequence surviving normalization shadows the
93-
* delta-allocated one and the merge-on-write merge retracts an updated row. The base delete sign,
94-
* version and commit tso are recomputed by the plan and pruned when unused, and the base row lsn is
95-
* the row identity IVM reads for DUP detail tables, so none of them is masked here.
89+
* The criterion is not "hidden column": it is a column whose name matches a column IVM owns and
90+
* that the delta rewrite reads back from the plan output by name. The base table's sequence
91+
* column ({@code Column.SEQUENCE_COL}) is the only such column today: IVM owns the MV's sequence
92+
* and several handlers select it with {@code findSlotByName}, so a base sequence surviving
93+
* normalization shadows the delta-allocated one. The two can even have different types (a base
94+
* DATETIME sequence against the MV's BIGINT), which turns the mistake into a rejected write.
95+
* <p>
96+
* The other hidden columns stay in place, deliberately. The base delete sign is referenced by the
97+
* {@code __DORIS_DELETE_SIGN__ = 0} filter directly above the scan, by ExprId; masking it would
98+
* leave that filter with a dangling slot. The MV's own delete sign is computed from the delta
99+
* factor rather than looked up, and for a merge-on-write base table a visible row's delete sign is
100+
* always 0. Version and commit tso are not read by name either -- the sink maps its hidden columns
101+
* positionally and defaults the missing ones -- and they are pruned when unused. The row lsn is
102+
* the row identity IVM reads for DUP detail tables.
96103
*/
97104
public static boolean isBaseScanStorageColumnToMask(String columnName) {
98105
return Column.SEQUENCE_COL.equalsIgnoreCase(columnName);
@@ -142,6 +149,12 @@ public static Expression getAppendOnlyDupHiddenSlotDefault(String columnName, Da
142149
: getCommonHiddenSlotDefault(columnName, targetType);
143150
}
144151

152+
/**
153+
* The value a common hidden slot takes when the plan writing it does not produce it. The data
154+
* type owns that value ({@link DataType#defaultValue()}), so a hidden slot of any type -- a
155+
* base table's sequence column can be a DATE/DATETIME, for example -- gets a non-null
156+
* placeholder instead of failing the rewrite; a type without one still fails loudly.
157+
*/
145158
public static Expression getCommonHiddenSlotDefault(String columnName, DataType targetType) {
146159
if (!isCommonHiddenSlot(columnName)) {
147160
throw new IvmException(IvmFailureReason.PLAN_REWRITE_FAILED,
@@ -161,17 +174,9 @@ public static Expression getCommonHiddenSlotDefault(String columnName, DataType
161174
if (defaultValue != null) {
162175
return defaultValue.castTo(targetType);
163176
}
164-
if (targetType.isNumericType()) {
165-
return new IntegerLiteral(0).castTo(targetType);
166-
} else if (targetType.isStringLikeType()) {
167-
return new VarcharLiteral("").castTo(targetType);
168-
} else if (targetType.isBitmapType()) {
169-
// Column.SKIP_BITMAP_COL is a bitmap type and uses bitmap_empty() as its default.
170-
return new BitmapEmpty();
171-
} else {
172-
throw new IvmException(IvmFailureReason.PLAN_REWRITE_FAILED,
173-
"no default value for IVM hidden slot: " + columnName + ", type=" + targetType);
174-
}
177+
return targetType.defaultValue().orElseThrow(() -> new IvmException(
178+
IvmFailureReason.PLAN_REWRITE_FAILED,
179+
"no default value for IVM hidden slot: " + columnName + ", type=" + targetType));
175180
}
176181

177182
/**

‎fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.doris.nereids.types;
1919

2020
import org.apache.doris.catalog.Type;
21+
import org.apache.doris.nereids.trees.expressions.Expression;
2122

2223
import com.google.common.base.Preconditions;
2324
import com.google.common.collect.ImmutableList;
@@ -27,6 +28,7 @@
2728
import java.util.Locale;
2829
import java.util.Map;
2930
import java.util.Objects;
31+
import java.util.Optional;
3032
import java.util.stream.Collectors;
3133

3234
/**
@@ -177,4 +179,9 @@ public int width() {
177179
public String toSql() {
178180
return toCatalogDataType().toSql();
179181
}
182+
183+
@Override
184+
public Optional<Expression> defaultValue() {
185+
return Optional.empty();
186+
}
180187
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,14 @@
1818
package org.apache.doris.nereids.types;
1919

2020
import org.apache.doris.catalog.Type;
21+
import org.apache.doris.nereids.trees.expressions.Expression;
22+
import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral;
23+
import org.apache.doris.nereids.trees.expressions.literal.Literal;
2124
import org.apache.doris.nereids.types.coercion.ComplexDataType;
2225

26+
import java.util.Collections;
2327
import java.util.Objects;
28+
import java.util.Optional;
2429

2530
/**
2631
* Array type in Nereids.
@@ -108,4 +113,9 @@ public int width() {
108113
public String toSql() {
109114
return "ARRAY<" + itemType.toSql() + ">";
110115
}
116+
117+
@Override
118+
public Optional<Expression> defaultValue() {
119+
return Optional.of(new ArrayLiteral(Collections.<Literal>emptyList(), this));
120+
}
111121
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/types/BigIntType.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,13 @@
1818
package org.apache.doris.nereids.types;
1919

2020
import org.apache.doris.catalog.Type;
21+
import org.apache.doris.nereids.trees.expressions.Expression;
22+
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
2123
import org.apache.doris.nereids.types.coercion.Int64OrLessType;
2224
import org.apache.doris.nereids.types.coercion.IntegralType;
2325

26+
import java.util.Optional;
27+
2428
/**
2529
* BigInt data type in Nereids.
2630
*/
@@ -67,4 +71,9 @@ public int width() {
6771
public int range() {
6872
return RANGE;
6973
}
74+
75+
@Override
76+
public Optional<Expression> defaultValue() {
77+
return Optional.of(new BigIntLiteral(0));
78+
}
7079
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/types/BitmapType.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,12 @@
1818
package org.apache.doris.nereids.types;
1919

2020
import org.apache.doris.catalog.Type;
21+
import org.apache.doris.nereids.trees.expressions.Expression;
22+
import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapEmpty;
2123
import org.apache.doris.nereids.types.coercion.PrimitiveType;
2224

25+
import java.util.Optional;
26+
2327
/**
2428
* Bitmap type in Nereids.
2529
*/
@@ -66,4 +70,9 @@ public int width() {
6670
public String toSql() {
6771
return "BITMAP";
6872
}
73+
74+
@Override
75+
public Optional<Expression> defaultValue() {
76+
return Optional.of(new BitmapEmpty());
77+
}
6978
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/types/BooleanType.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,12 @@
1818
package org.apache.doris.nereids.types;
1919

2020
import org.apache.doris.catalog.Type;
21+
import org.apache.doris.nereids.trees.expressions.Expression;
22+
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
2123
import org.apache.doris.nereids.types.coercion.PrimitiveType;
2224

25+
import java.util.Optional;
26+
2327
/**
2428
* Boolean type in Nereids.
2529
*/
@@ -56,4 +60,9 @@ public String simpleString() {
5660
public int width() {
5761
return WIDTH;
5862
}
63+
64+
@Override
65+
public Optional<Expression> defaultValue() {
66+
return Optional.of(BooleanLiteral.FALSE);
67+
}
5968
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/types/CharType.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,12 @@
1919

2020
import org.apache.doris.catalog.ScalarType;
2121
import org.apache.doris.catalog.Type;
22+
import org.apache.doris.nereids.trees.expressions.Expression;
23+
import org.apache.doris.nereids.trees.expressions.literal.CharLiteral;
2224
import org.apache.doris.nereids.types.coercion.CharacterType;
2325

2426
import java.util.Objects;
27+
import java.util.Optional;
2528

2629
/**
2730
* Char type in Nereids.
@@ -82,4 +85,9 @@ public String toSql() {
8285
}
8386
return "CHAR(" + len + ")";
8487
}
88+
89+
@Override
90+
public Optional<Expression> defaultValue() {
91+
return Optional.of(new CharLiteral("", width()));
92+
}
8593
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.apache.doris.nereids.annotation.Developing;
2424
import org.apache.doris.nereids.exceptions.AnalysisException;
2525
import org.apache.doris.nereids.parser.NereidsParser;
26+
import org.apache.doris.nereids.trees.expressions.Expression;
2627
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
2728
import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral;
2829
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
@@ -44,6 +45,7 @@
4445
import java.util.List;
4546
import java.util.Locale;
4647
import java.util.Map;
48+
import java.util.Optional;
4749
import java.util.Set;
4850
import java.util.function.Supplier;
4951
import java.util.stream.Collectors;
@@ -555,6 +557,14 @@ public DataType defaultConcreteType() {
555557
return this;
556558
}
557559

560+
/**
561+
* The non-null default value of this type: the value a NOT NULL column of the type is
562+
* initialized with when no value is supplied, and what a slot of this type holds when the
563+
* plan has no producer for it (e.g. an IVM hidden slot a source scan does not carry).
564+
* Empty for types that have no such value (state types, lambda, unsupported).
565+
*/
566+
public abstract Optional<Expression> defaultValue();
567+
558568
public boolean acceptsType(DataType other) {
559569
return sameType(other);
560570
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeType.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,15 @@
1919

2020
import org.apache.doris.catalog.Type;
2121
import org.apache.doris.common.Config;
22+
import org.apache.doris.nereids.trees.expressions.Expression;
23+
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
2224
import org.apache.doris.nereids.types.coercion.CharacterType;
2325
import org.apache.doris.nereids.types.coercion.DateLikeType;
2426

2527
import java.time.DateTimeException;
2628
import java.time.LocalDateTime;
2729
import java.time.temporal.ChronoUnit;
30+
import java.util.Optional;
2831

2932
/**
3033
* Datetime type in Nereids.
@@ -94,4 +97,9 @@ public double rangeLength(double high, double low) {
9497
return Double.POSITIVE_INFINITY;
9598
}
9699
}
100+
101+
@Override
102+
public Optional<Expression> defaultValue() {
103+
return Optional.of(new DateTimeLiteral(0, 1, 1, 0, 0, 0));
104+
}
97105
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@
2020
import org.apache.doris.catalog.ScalarType;
2121
import org.apache.doris.catalog.Type;
2222
import org.apache.doris.nereids.exceptions.AnalysisException;
23+
import org.apache.doris.nereids.trees.expressions.Expression;
2324
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
25+
import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
2426
import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
2527
import org.apache.doris.nereids.trees.expressions.literal.format.DateTimeChecker;
2628
import org.apache.doris.nereids.types.coercion.CharacterType;
@@ -32,6 +34,7 @@
3234
import java.time.LocalDateTime;
3335
import java.time.temporal.ChronoUnit;
3436
import java.util.Objects;
37+
import java.util.Optional;
3538

3639
/**
3740
* Datetime type in Nereids.
@@ -198,4 +201,9 @@ public double rangeLength(double high, double low) {
198201
return Double.POSITIVE_INFINITY;
199202
}
200203
}
204+
205+
@Override
206+
public Optional<Expression> defaultValue() {
207+
return Optional.of(new DateTimeV2Literal(this, 0, 1, 1, 0, 0, 0, 0));
208+
}
201209
}

0 commit comments

Comments
 (0)