Skip to content

Commit 09ff624

Browse files
committed
branch-3.0: [Enhancement] reject schema change on hidden columns (apache#53376)
Issue Number: close apache#53019 picked from apache#53376
1 parent 10791ef commit 09ff624

4 files changed

Lines changed: 125 additions & 28 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java‎

Lines changed: 38 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,12 @@ public String getValue() {
199199
private int clusterKeyId = -1;
200200
private Optional<GeneratedColumnInfo> generatedColumnInfo = Optional.empty();
201201
private Set<String> generatedColumnsThatReferToThis = new HashSet<>();
202+
// if add hidden column, must set enableAddHiddenColumn true
203+
private boolean enableAddHiddenColumn = false;
202204

205+
public void setEnableAddHiddenColumn(boolean enableAddHiddenColumn) {
206+
this.enableAddHiddenColumn = enableAddHiddenColumn;
207+
}
203208

204209
public ColumnDef(String name, TypeDef typeDef) {
205210
this(name, typeDef, false, null, ColumnNullableType.NOT_NULLABLE, DefaultValue.NOT_SET, "");
@@ -269,44 +274,63 @@ public ColumnDef(String name, TypeDef typeDef, boolean isKey, ColumnNullableType
269274
}
270275

271276
public static ColumnDef newDeleteSignColumnDef() {
272-
return new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, null,
277+
ColumnDef columnDef = new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, null,
273278
ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"),
274279
"doris delete flag hidden column", false, Optional.empty());
280+
columnDef.setEnableAddHiddenColumn(true);
281+
return columnDef;
275282
}
276283

277284
public static ColumnDef newDeleteSignColumnDef(AggregateType aggregateType) {
278-
return new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, aggregateType,
285+
ColumnDef columnDef = new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT),
286+
false, aggregateType,
279287
ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"),
280288
"doris delete flag hidden column", false, Optional.empty());
289+
columnDef.setEnableAddHiddenColumn(true);
290+
return columnDef;
281291
}
282292

283293
public static ColumnDef newSequenceColumnDef(Type type) {
284-
return new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false, null, ColumnNullableType.NULLABLE, -1,
294+
ColumnDef columnDef = new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false,
295+
null, ColumnNullableType.NULLABLE, -1,
285296
DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden column", false, Optional.empty());
297+
columnDef.setEnableAddHiddenColumn(true);
298+
return columnDef;
286299
}
287300

288301
public static ColumnDef newSequenceColumnDef(Type type, AggregateType aggregateType) {
289-
return new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false, aggregateType, ColumnNullableType.NULLABLE,
302+
ColumnDef columnDef = new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type),
303+
false, aggregateType, ColumnNullableType.NULLABLE,
290304
-1, DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden column", false,
291305
Optional.empty());
306+
columnDef.setEnableAddHiddenColumn(true);
307+
return columnDef;
292308
}
293309

294310
public static ColumnDef newRowStoreColumnDef(AggregateType aggregateType) {
295-
return new ColumnDef(Column.ROW_STORE_COL, TypeDef.create(PrimitiveType.STRING), false, aggregateType,
311+
ColumnDef columnDef = new ColumnDef(Column.ROW_STORE_COL, TypeDef.create(PrimitiveType.STRING),
312+
false, aggregateType,
296313
ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, ""),
297314
"doris row store hidden column", false, Optional.empty());
315+
columnDef.setEnableAddHiddenColumn(true);
316+
return columnDef;
298317
}
299318

300319
public static ColumnDef newVersionColumnDef() {
301-
return new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), false, null,
320+
ColumnDef columnDef = new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), false, null,
302321
ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"),
303322
"doris version hidden column", false, Optional.empty());
323+
columnDef.setEnableAddHiddenColumn(true);
324+
return columnDef;
304325
}
305326

306327
public static ColumnDef newVersionColumnDef(AggregateType aggregateType) {
307-
return new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), false, aggregateType,
328+
ColumnDef columnDef = new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT),
329+
false, aggregateType,
308330
ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"),
309331
"doris version hidden column", false, Optional.empty());
332+
columnDef.setEnableAddHiddenColumn(true);
333+
return columnDef;
310334
}
311335

312336
public boolean isAllowNull() {
@@ -369,7 +393,13 @@ public void analyze(boolean isOlap) throws AnalysisException {
369393
if (name == null || typeDef == null) {
370394
throw new AnalysisException("No column name or column type in column definition.");
371395
}
372-
FeNameFormat.checkColumnName(name);
396+
// if enableAddHiddenColumn is true, can add hidden column.
397+
// So does not check if the column name starts with __DORIS_
398+
if (enableAddHiddenColumn) {
399+
FeNameFormat.checkColumnNameBypassHiddenColumn(name);
400+
} else {
401+
FeNameFormat.checkColumnName(name);
402+
}
373403
FeNameFormat.checkColumnCommentLength(comment);
374404

375405
typeDef.analyze(null);

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@
6060
*/
6161
public class Column implements GsonPostProcessable {
6262
private static final Logger LOG = LogManager.getLogger(Column.class);
63+
public static final String HIDDEN_COLUMN_PREFIX = "__DORIS_";
6364
// NOTE: you should name hidden column start with '__DORIS_' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
6465
public static final String DELETE_SIGN = "__DORIS_DELETE_SIGN__";
6566
public static final String WHERE_SIGN = "__DORIS_WHERE_SIGN__";

‎fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.apache.doris.alter.SchemaChangeHandler;
2121
import org.apache.doris.analysis.CreateMaterializedViewStmt;
2222
import org.apache.doris.analysis.ResourceTypeEnum;
23+
import org.apache.doris.catalog.Column;
2324
import org.apache.doris.datasource.InternalCatalog;
2425
import org.apache.doris.mysql.privilege.Role;
2526
import org.apache.doris.mysql.privilege.RoleManager;
@@ -86,18 +87,29 @@ public static void checkPartitionName(String partitionName) throws AnalysisExcep
8687
}
8788

8889
public static void checkColumnName(String columnName) throws AnalysisException {
90+
// if need check another column name prefix, add in `checkColumnNameBypassHiddenColumn`
91+
checkColumnNameBypassHiddenColumn(columnName);
92+
checkColumnNamePrefix(columnName, Column.HIDDEN_COLUMN_PREFIX);
93+
}
94+
95+
public static void checkColumnNameBypassHiddenColumn(String columnName) throws AnalysisException {
8996
if (Strings.isNullOrEmpty(columnName) || !columnName.matches(getColumnNameRegex())) {
9097
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME,
9198
columnName, getColumnNameRegex());
9299
}
93-
if (columnName.startsWith(SchemaChangeHandler.SHADOW_NAME_PREFIX)) {
94-
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME,
95-
columnName, getColumnNameRegex());
100+
checkColumnNamePrefix(columnName, SchemaChangeHandler.SHADOW_NAME_PREFIX);
101+
checkColumnNamePrefix(columnName, CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX);
102+
checkColumnNamePrefix(columnName, CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX);
103+
}
104+
105+
private static void checkColumnNamePrefix(String columnName, String prefix) throws AnalysisException {
106+
int prefixLength = prefix.length();
107+
if (columnName.length() < prefixLength) {
108+
return;
96109
}
97-
if (columnName.startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX)
98-
|| columnName.startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX)) {
110+
if (columnName.substring(0, prefixLength).equalsIgnoreCase(prefix)) {
99111
throw new AnalysisException(
100-
"Incorrect column name " + columnName + ", column name can't start with 'mv_'/'mva_'");
112+
"Incorrect column name " + columnName + ", column name can't start with '" + prefix + "'");
101113
}
102114
}
103115

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java‎

Lines changed: 68 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ public class ColumnDefinition {
6868
private int clusterKeyId = -1;
6969
private Optional<GeneratedColumnDesc> generatedColumnDesc = Optional.empty();
7070
private Set<String> generatedColumnsThatReferToThis = new HashSet<>();
71+
// if add hidden column, must set enableAddHiddenColumn true
72+
private boolean enableAddHiddenColumn = false;
7173

7274
public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType aggType, boolean isNullable,
7375
Optional<DefaultValue> defaultValue, String comment) {
@@ -231,7 +233,14 @@ private void checkKeyColumnType(boolean isOlap) {
231233
public void validate(boolean isOlap, Set<String> keysSet, Set<String> clusterKeySet, boolean isEnableMergeOnWrite,
232234
KeysType keysType) {
233235
try {
234-
FeNameFormat.checkColumnName(name);
236+
// if enableAddHiddenColumn is true, can add hidden column.
237+
// So does not check if the column name starts with __DORIS_
238+
if (enableAddHiddenColumn) {
239+
FeNameFormat.checkColumnNameBypassHiddenColumn(name);
240+
} else {
241+
FeNameFormat.checkColumnName(name);
242+
}
243+
235244
FeNameFormat.checkColumnCommentLength(comment);
236245
} catch (Exception e) {
237246
throw new AnalysisException(e.getMessage(), e);
@@ -674,35 +683,76 @@ public Column translateToCatalogStyle() {
674683
return column;
675684
}
676685

677-
// hidden column
686+
/**
687+
* add hidden column
688+
*/
678689
public static ColumnDefinition newDeleteSignColumnDefinition() {
679-
return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, null, false,
680-
Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris delete flag hidden column", false);
690+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, null,
691+
false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
692+
"doris delete flag hidden column", false);
693+
columnDefinition.setEnableAddHiddenColumn(true);
694+
695+
return columnDefinition;
681696
}
682697

698+
/**
699+
* add hidden column
700+
*/
683701
public static ColumnDefinition newDeleteSignColumnDefinition(AggregateType aggregateType) {
684-
return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, aggregateType, false,
685-
Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris delete flag hidden column", false);
702+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false,
703+
aggregateType, false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
704+
"doris delete flag hidden column", false);
705+
columnDefinition.setEnableAddHiddenColumn(true);
706+
707+
return columnDefinition;
686708
}
687709

710+
/**
711+
* add hidden column
712+
*/
688713
public static ColumnDefinition newSequenceColumnDefinition(DataType type) {
689-
return new ColumnDefinition(Column.SEQUENCE_COL, type, false, null, true,
690-
Optional.empty(), "sequence column hidden column", false);
714+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.SEQUENCE_COL, type, false, null,
715+
true, Optional.empty(),
716+
"sequence column hidden column", false);
717+
columnDefinition.setEnableAddHiddenColumn(true);
718+
719+
return columnDefinition;
691720
}
692721

722+
/**
723+
* add hidden column
724+
*/
693725
public static ColumnDefinition newSequenceColumnDefinition(DataType type, AggregateType aggregateType) {
694-
return new ColumnDefinition(Column.SEQUENCE_COL, type, false, aggregateType, true,
695-
Optional.empty(), "sequence column hidden column", false);
726+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.SEQUENCE_COL, type, false, aggregateType,
727+
true, Optional.empty(),
728+
"sequence column hidden column", false);
729+
columnDefinition.setEnableAddHiddenColumn(true);
730+
731+
return columnDefinition;
696732
}
697733

734+
/**
735+
* add hidden column
736+
*/
698737
public static ColumnDefinition newRowStoreColumnDefinition(AggregateType aggregateType) {
699-
return new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false, aggregateType, false,
700-
Optional.of(new DefaultValue("")), "doris row store hidden column", false);
738+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false,
739+
aggregateType, false, Optional.of(new DefaultValue("")),
740+
"doris row store hidden column", false);
741+
columnDefinition.setEnableAddHiddenColumn(true);
742+
743+
return columnDefinition;
701744
}
702745

746+
/**
747+
* add hidden column
748+
*/
703749
public static ColumnDefinition newVersionColumnDefinition(AggregateType aggregateType) {
704-
return new ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE, false, aggregateType, false,
705-
Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris version hidden column", false);
750+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE, false,
751+
aggregateType, false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
752+
"doris version hidden column", false);
753+
columnDefinition.setEnableAddHiddenColumn(true);
754+
755+
return columnDefinition;
706756
}
707757

708758
public Optional<GeneratedColumnDesc> getGeneratedColumnDesc() {
@@ -717,6 +767,10 @@ public void addGeneratedColumnsThatReferToThis(List<String> list) {
717767
generatedColumnsThatReferToThis.addAll(list);
718768
}
719769

770+
public void setEnableAddHiddenColumn(boolean enableAddHiddenColumn) {
771+
this.enableAddHiddenColumn = enableAddHiddenColumn;
772+
}
773+
720774
private void validateGeneratedColumnInfo() {
721775
// for generated column
722776
if (generatedColumnDesc.isPresent()) {

0 commit comments

Comments
 (0)