Skip to content

Commit f95d23e

Browse files
committed
branch-3.1: [Enhancement] reject schema change on hidden columns (apache#53376)
Issue Number: close apache#53019 picked from apache#53376
1 parent 78689d7 commit f95d23e

4 files changed

Lines changed: 115 additions & 44 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java‎

Lines changed: 45 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,8 @@ public String getValue() {
199199
private int clusterKeyId = -1;
200200
private Optional<GeneratedColumnInfo> generatedColumnInfo = Optional.empty();
201201
private Set<String> generatedColumnsThatReferToThis = new HashSet<>();
202+
// if add hidden column, must set enableAddHiddenColumn true
203+
private boolean enableAddHiddenColumn = false;
202204

203205

204206
public ColumnDef(String name, TypeDef typeDef) {
@@ -262,57 +264,83 @@ public ColumnDef(String name, TypeDef typeDef, boolean isKey, AggregateType aggr
262264
this.visible = visible;
263265
}
264266

267+
public void setEnableAddHiddenColumn(boolean enableAddHiddenColumn) {
268+
this.enableAddHiddenColumn = enableAddHiddenColumn;
269+
}
270+
265271
public ColumnDef(String name, TypeDef typeDef, boolean isKey, ColumnNullableType nullableType, String comment,
266272
Optional<GeneratedColumnInfo> generatedColumnInfo) {
267273
this(name, typeDef, isKey, null, nullableType, -1, DefaultValue.NOT_SET,
268274
comment, true, generatedColumnInfo);
269275
}
270276

271277
public static ColumnDef newDeleteSignColumnDef() {
272-
return new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, null,
273-
ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"),
278+
ColumnDef columnDef = new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, null,
279+
ColumnNullableType.NOT_NULLABLE, -1, new DefaultValue(true, "0"),
274280
"doris delete flag hidden column", false, Optional.empty());
281+
columnDef.setEnableAddHiddenColumn(true);
282+
return columnDef;
275283
}
276284

277285
public static ColumnDef newDeleteSignColumnDef(AggregateType aggregateType) {
278-
return new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, aggregateType,
279-
ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"),
286+
ColumnDef columnDef = new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false,
287+
aggregateType,
288+
ColumnNullableType.NOT_NULLABLE, -1, new DefaultValue(true, "0"),
280289
"doris delete flag hidden column", false, Optional.empty());
290+
columnDef.setEnableAddHiddenColumn(true);
291+
return columnDef;
281292
}
282293

283294
public static ColumnDef newSequenceColumnDef(Type type) {
284-
return new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false, null, ColumnNullableType.NULLABLE, -1,
295+
ColumnDef columnDef = new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type),
296+
false, null, ColumnNullableType.NULLABLE, -1,
285297
DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden column", false, Optional.empty());
298+
columnDef.setEnableAddHiddenColumn(true);
299+
return columnDef;
286300
}
287301

288302
public static ColumnDef newSequenceColumnDef(Type type, AggregateType aggregateType) {
289-
return new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false, aggregateType, ColumnNullableType.NULLABLE,
303+
ColumnDef columnDef = new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type),
304+
false, aggregateType, ColumnNullableType.NULLABLE,
290305
-1, DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden column", false,
291306
Optional.empty());
307+
columnDef.setEnableAddHiddenColumn(true);
308+
return columnDef;
292309
}
293310

294311
public static ColumnDef newRowStoreColumnDef(AggregateType aggregateType) {
295-
return new ColumnDef(Column.ROW_STORE_COL, TypeDef.create(PrimitiveType.STRING), false, aggregateType,
312+
ColumnDef columnDef = new ColumnDef(Column.ROW_STORE_COL, TypeDef.create(PrimitiveType.STRING),
313+
false, aggregateType,
296314
ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, ""),
297315
"doris row store hidden column", false, Optional.empty());
316+
columnDef.setEnableAddHiddenColumn(true);
317+
return columnDef;
298318
}
299319

300320
public static ColumnDef newVersionColumnDef() {
301-
return new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), false, null,
321+
ColumnDef columnDef = new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), false, null,
302322
ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"),
303323
"doris version hidden column", false, Optional.empty());
324+
columnDef.setEnableAddHiddenColumn(true);
325+
return columnDef;
304326
}
305327

306328
public static ColumnDef newVersionColumnDef(AggregateType aggregateType) {
307-
return new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), false, aggregateType,
329+
ColumnDef columnDef = new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT),
330+
false, aggregateType,
308331
ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"),
309332
"doris version hidden column", false, Optional.empty());
333+
columnDef.setEnableAddHiddenColumn(true);
334+
return columnDef;
310335
}
311336

312337
public static ColumnDef newSkipBitmapColumnDef(AggregateType aggregateType) {
313-
return new ColumnDef(Column.SKIP_BITMAP_COL, TypeDef.create(PrimitiveType.BITMAP), false, aggregateType,
338+
ColumnDef columnDef = new ColumnDef(Column.SKIP_BITMAP_COL, TypeDef.create(PrimitiveType.BITMAP),
339+
false, aggregateType,
314340
ColumnNullableType.NOT_NULLABLE, -1, DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE,
315341
"doris skip bitmap hidden column", false, Optional.empty());
342+
columnDef.setEnableAddHiddenColumn(true);
343+
return columnDef;
316344
}
317345

318346
public boolean isAllowNull() {
@@ -375,7 +403,13 @@ public void analyze(boolean isOlap) throws AnalysisException {
375403
if (name == null || typeDef == null) {
376404
throw new AnalysisException("No column name or column type in column definition.");
377405
}
378-
FeNameFormat.checkColumnName(name);
406+
407+
if (enableAddHiddenColumn) {
408+
FeNameFormat.checkColumnNameBypassHiddenColumn(name);
409+
} else {
410+
FeNameFormat.checkColumnName(name);
411+
}
412+
379413
FeNameFormat.checkColumnCommentLength(comment);
380414

381415
typeDef.analyze(null);

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
*/
6363
public class Column implements GsonPostProcessable {
6464
private static final Logger LOG = LogManager.getLogger(Column.class);
65+
public static final String HIDDEN_COLUMN_PREFIX = "__DORIS_";
6566
// NOTE: you should name hidden column start with '__DORIS_' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
6667
public static final String DELETE_SIGN = "__DORIS_DELETE_SIGN__";
6768
public static final String WHERE_SIGN = "__DORIS_WHERE_SIGN__";

‎fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.apache.doris.alter.SchemaChangeHandler;
2121
import org.apache.doris.analysis.CreateMaterializedViewStmt;
2222
import org.apache.doris.analysis.ResourceTypeEnum;
23+
import org.apache.doris.catalog.Column;
2324
import org.apache.doris.datasource.InternalCatalog;
2425
import org.apache.doris.mysql.privilege.Role;
2526
import org.apache.doris.mysql.privilege.RoleManager;
@@ -93,18 +94,29 @@ public static void checkPartitionName(String partitionName) throws AnalysisExcep
9394
}
9495

9596
public static void checkColumnName(String columnName) throws AnalysisException {
97+
// if need check another column name prefix, add in `checkColumnNameBypassHiddenColumn`
98+
checkColumnNameBypassHiddenColumn(columnName);
99+
checkColumnNamePrefix(columnName, Column.HIDDEN_COLUMN_PREFIX);
100+
}
101+
102+
public static void checkColumnNameBypassHiddenColumn(String columnName) throws AnalysisException {
96103
if (Strings.isNullOrEmpty(columnName) || !columnName.matches(getColumnNameRegex())) {
97104
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME,
98105
columnName, getColumnNameRegex());
99106
}
100-
if (columnName.startsWith(SchemaChangeHandler.SHADOW_NAME_PREFIX)) {
101-
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME,
102-
columnName, getColumnNameRegex());
107+
checkColumnNamePrefix(columnName, SchemaChangeHandler.SHADOW_NAME_PREFIX);
108+
checkColumnNamePrefix(columnName, CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX);
109+
checkColumnNamePrefix(columnName, CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX);
110+
}
111+
112+
private static void checkColumnNamePrefix(String columnName, String prefix) throws AnalysisException {
113+
int prefixLength = prefix.length();
114+
if (columnName.length() < prefixLength) {
115+
return;
103116
}
104-
if (columnName.startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX)
105-
|| columnName.startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX)) {
117+
if (columnName.substring(0, prefixLength).equalsIgnoreCase(prefix)) {
106118
throw new AnalysisException(
107-
"Incorrect column name " + columnName + ", column name can't start with 'mv_'/'mva_'");
119+
"Incorrect column name " + columnName + ", column name can't start with '" + prefix + "'");
108120
}
109121
}
110122

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java‎

Lines changed: 51 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ public class ColumnDefinition {
6969
private int clusterKeyId = -1;
7070
private Optional<GeneratedColumnDesc> generatedColumnDesc = Optional.empty();
7171
private Set<String> generatedColumnsThatReferToThis = new HashSet<>();
72+
// if add hidden column, must set enableAddHiddenColumn true
73+
private boolean enableAddHiddenColumn = false;
7274

7375
public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType aggType, boolean isNullable,
7476
Optional<DefaultValue> defaultValue, String comment) {
@@ -232,7 +234,14 @@ private void checkKeyColumnType(boolean isOlap) {
232234
public void validate(boolean isOlap, Set<String> keysSet, Set<String> clusterKeySet, boolean isEnableMergeOnWrite,
233235
KeysType keysType) {
234236
try {
235-
FeNameFormat.checkColumnName(name);
237+
// if enableAddHiddenColumn is true, can add hidden column.
238+
// So does not check if the column name starts with __DORIS_
239+
if (enableAddHiddenColumn) {
240+
FeNameFormat.checkColumnNameBypassHiddenColumn(name);
241+
} else {
242+
FeNameFormat.checkColumnName(name);
243+
}
244+
236245
FeNameFormat.checkColumnCommentLength(comment);
237246
} catch (Exception e) {
238247
throw new AnalysisException(e.getMessage(), e);
@@ -689,43 +698,54 @@ public Column translateToCatalogStyle() {
689698
return column;
690699
}
691700

692-
// hidden column
693-
public static ColumnDefinition newDeleteSignColumnDefinition() {
694-
return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, null, false,
695-
Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris delete flag hidden column", false);
696-
}
697-
701+
/**
702+
* add hidden column
703+
*/
698704
public static ColumnDefinition newDeleteSignColumnDefinition(AggregateType aggregateType) {
699-
return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, aggregateType, false,
700-
Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris delete flag hidden column", false);
701-
}
705+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false,
706+
aggregateType, false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
707+
"doris delete flag hidden column", false);
708+
columnDefinition.setEnableAddHiddenColumn(true);
702709

703-
public static ColumnDefinition newSequenceColumnDefinition(DataType type) {
704-
return new ColumnDefinition(Column.SEQUENCE_COL, type, false, null, true,
705-
Optional.empty(), "sequence column hidden column", false);
706-
}
707-
708-
public static ColumnDefinition newSequenceColumnDefinition(DataType type, AggregateType aggregateType) {
709-
return new ColumnDefinition(Column.SEQUENCE_COL, type, false, aggregateType, true,
710-
Optional.empty(), "sequence column hidden column", false);
710+
return columnDefinition;
711711
}
712712

713+
/**
714+
* add hidden column
715+
*/
713716
public static ColumnDefinition newRowStoreColumnDefinition(AggregateType aggregateType) {
714-
return new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false, aggregateType, false,
715-
Optional.of(new DefaultValue("")), "doris row store hidden column", false);
717+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false,
718+
aggregateType, false, Optional.of(new DefaultValue("")),
719+
"doris row store hidden column", false);
720+
columnDefinition.setEnableAddHiddenColumn(true);
721+
722+
return columnDefinition;
716723
}
717724

725+
/**
726+
* add hidden column
727+
*/
718728
public static ColumnDefinition newVersionColumnDefinition(AggregateType aggregateType) {
719-
return new ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE, false, aggregateType, false,
720-
Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris version hidden column", false);
729+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE, false,
730+
aggregateType, false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
731+
"doris version hidden column", false);
732+
columnDefinition.setEnableAddHiddenColumn(true);
733+
734+
return columnDefinition;
721735
}
722736

723-
// used in CreateTableInfo.validate(), specify the default value as DefaultValue.NULL_DEFAULT_VALUE
724-
// becasue ColumnDefinition.validate() will check that bitmap type column don't set default value
725-
// and then set the default value of that column to bitmap_empty()
737+
/**
738+
* used in CreateTableInfo.validate(), specify the default value as DefaultValue.NULL_DEFAULT_VALUE
739+
* becasue ColumnDefinition.validate() will check that bitmap type column don't set default value
740+
* and then set the default value of that column to bitmap_empty()
741+
*/
726742
public static ColumnDefinition newSkipBitmapColumnDef(AggregateType aggregateType) {
727-
return new ColumnDefinition(Column.SKIP_BITMAP_COL, BitmapType.INSTANCE, false, aggregateType, false,
728-
Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE), "doris skip bitmap hidden column", false);
743+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.SKIP_BITMAP_COL, BitmapType.INSTANCE, false,
744+
aggregateType, false, Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE),
745+
"doris skip bitmap hidden column", false);
746+
columnDefinition.setEnableAddHiddenColumn(true);
747+
748+
return columnDefinition;
729749
}
730750

731751
public Optional<GeneratedColumnDesc> getGeneratedColumnDesc() {
@@ -740,6 +760,10 @@ public void addGeneratedColumnsThatReferToThis(List<String> list) {
740760
generatedColumnsThatReferToThis.addAll(list);
741761
}
742762

763+
public void setEnableAddHiddenColumn(boolean enableAddHiddenColumn) {
764+
this.enableAddHiddenColumn = enableAddHiddenColumn;
765+
}
766+
743767
private void validateGeneratedColumnInfo() {
744768
// for generated column
745769
if (generatedColumnDesc.isPresent()) {

0 commit comments

Comments
 (0)