Skip to content

Commit 5708caa

Browse files
committed
branch-2.1: [Enhancement] reject schema change on hidden columns (apache#53376)
Issue Number: close apache#53019 picked from apache#53376
1 parent e7ee762 commit 5708caa

4 files changed

Lines changed: 128 additions & 28 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java‎

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,12 @@ public String getValue() {
185185
private String comment;
186186
private boolean visible;
187187
private int clusterKeyId = -1;
188+
// if add hidden column, must set enableAddHiddenColumn true
189+
private boolean enableAddHiddenColumn = false;
190+
191+
public void setEnableAddHiddenColumn(boolean enableAddHiddenColumn) {
192+
this.enableAddHiddenColumn = enableAddHiddenColumn;
193+
}
188194

189195
public ColumnDef(String name, TypeDef typeDef) {
190196
this(name, typeDef, false, null, false, -1, DefaultValue.NOT_SET, "");
@@ -219,40 +225,59 @@ public ColumnDef(String name, TypeDef typeDef, boolean isKey, AggregateType aggr
219225
}
220226

221227
public static ColumnDef newDeleteSignColumnDef() {
222-
return new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, null, false,
228+
ColumnDef columnDef = new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT),
229+
false, null, false,
223230
-1, new ColumnDef.DefaultValue(true, "0"), "doris delete flag hidden column", false);
231+
columnDef.setEnableAddHiddenColumn(true);
232+
return columnDef;
224233
}
225234

226235
public static ColumnDef newDeleteSignColumnDef(AggregateType aggregateType) {
227-
return new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, aggregateType, false,
236+
ColumnDef columnDef = new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT),
237+
false, aggregateType, false,
228238
-1, new ColumnDef.DefaultValue(true, "0"), "doris delete flag hidden column", false);
239+
columnDef.setEnableAddHiddenColumn(true);
240+
return columnDef;
229241
}
230242

231243
public static ColumnDef newSequenceColumnDef(Type type) {
232-
return new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false, null, true,
244+
ColumnDef columnDef = new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type),
245+
false, null, true,
233246
-1, DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden column", false);
247+
columnDef.setEnableAddHiddenColumn(true);
248+
return columnDef;
234249
}
235250

236251
public static ColumnDef newSequenceColumnDef(Type type, AggregateType aggregateType) {
237-
return new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false,
252+
ColumnDef columnDef = new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false,
238253
aggregateType, true, -1, DefaultValue.NULL_DEFAULT_VALUE,
239254
"sequence column hidden column", false);
255+
columnDef.setEnableAddHiddenColumn(true);
256+
return columnDef;
240257
}
241258

242259
public static ColumnDef newRowStoreColumnDef(AggregateType aggregateType) {
243-
return new ColumnDef(Column.ROW_STORE_COL, TypeDef.create(PrimitiveType.STRING), false,
260+
ColumnDef columnDef = new ColumnDef(Column.ROW_STORE_COL, TypeDef.create(PrimitiveType.STRING), false,
244261
aggregateType, false, -1,
245262
new ColumnDef.DefaultValue(true, ""), "doris row store hidden column", false);
263+
columnDef.setEnableAddHiddenColumn(true);
264+
return columnDef;
246265
}
247266

248267
public static ColumnDef newVersionColumnDef() {
249-
return new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), false, null, false, -1,
268+
ColumnDef columnDef = new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT),
269+
false, null, false, -1,
250270
new ColumnDef.DefaultValue(true, "0"), "doris version hidden column", false);
271+
columnDef.setEnableAddHiddenColumn(true);
272+
return columnDef;
251273
}
252274

253275
public static ColumnDef newVersionColumnDef(AggregateType aggregateType) {
254-
return new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), false, aggregateType, false,
276+
ColumnDef columnDef = new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT),
277+
false, aggregateType, false,
255278
-1, new ColumnDef.DefaultValue(true, "0"), "doris version hidden column", false);
279+
columnDef.setEnableAddHiddenColumn(true);
280+
return columnDef;
256281
}
257282

258283
public boolean isAllowNull() {
@@ -311,7 +336,13 @@ public void analyze(boolean isOlap) throws AnalysisException {
311336
if (name == null || typeDef == null) {
312337
throw new AnalysisException("No column name or column type in column definition.");
313338
}
314-
FeNameFormat.checkColumnName(name);
339+
// if enableAddHiddenColumn is true, can add hidden column.
340+
// So does not check if the column name starts with __DORIS_
341+
if (enableAddHiddenColumn) {
342+
FeNameFormat.checkColumnNameBypassHiddenColumn(name);
343+
} else {
344+
FeNameFormat.checkColumnName(name);
345+
}
315346

316347
typeDef.analyze(null);
317348

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
*/
5656
public class Column implements Writable, GsonPostProcessable {
5757
private static final Logger LOG = LogManager.getLogger(Column.class);
58+
public static final String HIDDEN_COLUMN_PREFIX = "__DORIS_";
5859
// NOTE: you should name hidden column start with '__DORIS_' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
5960
public static final String DELETE_SIGN = "__DORIS_DELETE_SIGN__";
6061
public static final String WHERE_SIGN = "__DORIS_WHERE_SIGN__";

‎fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
import org.apache.doris.alter.SchemaChangeHandler;
2121
import org.apache.doris.analysis.CreateMaterializedViewStmt;
22+
import org.apache.doris.catalog.Column;
2223
import org.apache.doris.datasource.InternalCatalog;
2324
import org.apache.doris.mysql.privilege.Role;
2425
import org.apache.doris.mysql.privilege.RoleManager;
@@ -85,18 +86,29 @@ public static void checkPartitionName(String partitionName) throws AnalysisExcep
8586
}
8687

8788
public static void checkColumnName(String columnName) throws AnalysisException {
89+
// if need check another column name prefix, add in `checkColumnNameBypassHiddenColumn`
90+
checkColumnNameBypassHiddenColumn(columnName);
91+
checkColumnNamePrefix(columnName, Column.HIDDEN_COLUMN_PREFIX);
92+
}
93+
94+
public static void checkColumnNameBypassHiddenColumn(String columnName) throws AnalysisException {
8895
if (Strings.isNullOrEmpty(columnName) || !columnName.matches(getColumnNameRegex())) {
8996
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME,
9097
columnName, getColumnNameRegex());
9198
}
92-
if (columnName.startsWith(SchemaChangeHandler.SHADOW_NAME_PREFIX)) {
93-
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME,
94-
columnName, getColumnNameRegex());
99+
checkColumnNamePrefix(columnName, SchemaChangeHandler.SHADOW_NAME_PREFIX);
100+
checkColumnNamePrefix(columnName, CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX);
101+
checkColumnNamePrefix(columnName, CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX);
102+
}
103+
104+
private static void checkColumnNamePrefix(String columnName, String prefix) throws AnalysisException {
105+
int prefixLength = prefix.length();
106+
if (columnName.length() < prefixLength) {
107+
return;
95108
}
96-
if (columnName.startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX)
97-
|| columnName.startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX)) {
109+
if (columnName.substring(0, prefixLength).equalsIgnoreCase(prefix)) {
98110
throw new AnalysisException(
99-
"Incorrect column name " + columnName + ", column name can't start with 'mv_'/'mva_'");
111+
"Incorrect column name " + columnName + ", column name can't start with '" + prefix + "'");
100112
}
101113
}
102114

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java‎

Lines changed: 70 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@ public class ColumnDefinition {
6565
private boolean aggTypeImplicit = false;
6666
private long autoIncInitValue = -1;
6767
private int clusterKeyId = -1;
68+
// if add hidden column, must set enableAddHiddenColumn true
69+
private boolean enableAddHiddenColumn = false;
6870

6971
public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType aggType, boolean isNullable,
7072
Optional<DefaultValue> defaultValue, String comment) {
@@ -131,6 +133,10 @@ public AggregateType getAggType() {
131133
return aggType;
132134
}
133135

136+
public void setEnableAddHiddenColumn(boolean enableAddHiddenColumn) {
137+
this.enableAddHiddenColumn = enableAddHiddenColumn;
138+
}
139+
134140
public void setAggType(AggregateType aggType) {
135141
this.aggType = aggType;
136142
}
@@ -180,7 +186,13 @@ private DataType updateCharacterTypeLength(DataType dataType) {
180186
*/
181187
public void validate(boolean isOlap, Set<String> keysSet, boolean isEnableMergeOnWrite, KeysType keysType) {
182188
try {
183-
FeNameFormat.checkColumnName(name);
189+
// if enableAddHiddenColumn is true, can add hidden column.
190+
// So does not check if the column name starts with __DORIS_
191+
if (enableAddHiddenColumn) {
192+
FeNameFormat.checkColumnNameBypassHiddenColumn(name);
193+
} else {
194+
FeNameFormat.checkColumnName(name);
195+
}
184196
} catch (Exception e) {
185197
throw new AnalysisException(e.getMessage(), e);
186198
}
@@ -639,35 +651,79 @@ public Column translateToCatalogStyle() {
639651
return column;
640652
}
641653

642-
// hidden column
654+
/**
655+
* add hidden column
656+
*/
643657
public static ColumnDefinition newDeleteSignColumnDefinition() {
644-
return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, null, false,
645-
Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris delete flag hidden column", false);
658+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, null,
659+
false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
660+
"doris delete flag hidden column", false);
661+
columnDefinition.setEnableAddHiddenColumn(true);
662+
663+
return columnDefinition;
646664
}
647665

666+
/**
667+
* add hidden column
668+
*/
648669
public static ColumnDefinition newDeleteSignColumnDefinition(AggregateType aggregateType) {
649-
return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, aggregateType, false,
650-
Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris delete flag hidden column", false);
670+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false,
671+
aggregateType, false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
672+
"doris delete flag hidden column", false);
673+
columnDefinition.setEnableAddHiddenColumn(true);
674+
675+
return columnDefinition;
651676
}
652677

678+
/**
679+
* add hidden column
680+
*/
653681
public static ColumnDefinition newSequenceColumnDefinition(DataType type) {
654-
return new ColumnDefinition(Column.SEQUENCE_COL, type, false, null, true,
655-
Optional.empty(), "sequence column hidden column", false);
682+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.SEQUENCE_COL, type, false, null,
683+
true, Optional.empty(),
684+
"sequence column hidden column", false);
685+
columnDefinition.setEnableAddHiddenColumn(true);
686+
687+
return columnDefinition;
656688
}
657689

690+
/**
691+
* add hidden column
692+
*/
658693
public static ColumnDefinition newSequenceColumnDefinition(DataType type, AggregateType aggregateType) {
659-
return new ColumnDefinition(Column.SEQUENCE_COL, type, false, aggregateType, true,
660-
Optional.empty(), "sequence column hidden column", false);
694+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.SEQUENCE_COL, type, false, aggregateType,
695+
true, Optional.empty(),
696+
"sequence column hidden column", false);
697+
columnDefinition.setEnableAddHiddenColumn(true);
698+
699+
return columnDefinition;
661700
}
662701

702+
/**
703+
* add hidden column
704+
*/
663705
public static ColumnDefinition newRowStoreColumnDefinition(AggregateType aggregateType) {
664-
return new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false, aggregateType, false,
665-
Optional.of(new DefaultValue("")), "doris row store hidden column", false);
706+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false,
707+
aggregateType, false, Optional.of(new DefaultValue("")),
708+
"doris row store hidden column", false);
709+
columnDefinition.setEnableAddHiddenColumn(true);
710+
711+
return columnDefinition;
666712
}
667713

714+
/**
715+
* add hidden column
716+
*/
668717
public static ColumnDefinition newVersionColumnDefinition(AggregateType aggregateType) {
669-
return new ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE, false, aggregateType, false,
670-
Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris version hidden column", false);
718+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE, false,
719+
aggregateType, false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
720+
"doris version hidden column", false);
721+
columnDefinition.setEnableAddHiddenColumn(true);
722+
723+
return columnDefinition;
671724
}
672725

726+
public long getAutoIncInitValue() {
727+
return autoIncInitValue;
728+
}
673729
}

0 commit comments

Comments
 (0)