Skip to content

Commit 57e4b91

Browse files
authored
[Enhancement] reject schema change on hidden columns (apache#53376)
Issue Number: close apache#53019
1 parent 8289b0a commit 57e4b91

6 files changed

Lines changed: 174 additions & 19 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
*/
5959
public class Column implements GsonPostProcessable {
6060
private static final Logger LOG = LogManager.getLogger(Column.class);
61+
public static final String HIDDEN_COLUMN_PREFIX = "__DORIS_";
6162
// NOTE: you should name hidden column start with '__DORIS_' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
6263
public static final String DELETE_SIGN = "__DORIS_DELETE_SIGN__";
6364
public static final String WHERE_SIGN = "__DORIS_WHERE_SIGN__";

‎fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.apache.doris.alter.SchemaChangeHandler;
2121
import org.apache.doris.analysis.CreateMaterializedViewStmt;
2222
import org.apache.doris.analysis.ResourceTypeEnum;
23+
import org.apache.doris.catalog.Column;
2324
import org.apache.doris.datasource.InternalCatalog;
2425
import org.apache.doris.mysql.privilege.Role;
2526
import org.apache.doris.mysql.privilege.RoleManager;
@@ -94,6 +95,12 @@ public static void checkPartitionName(String partitionName) throws AnalysisExcep
9495
}
9596

9697
public static void checkColumnName(String columnName) throws AnalysisException {
98+
// if need check another column name prefix, add in `checkColumnNameBypassHiddenColumn`
99+
checkColumnNameBypassHiddenColumn(columnName);
100+
checkColumnNamePrefix(columnName, Column.HIDDEN_COLUMN_PREFIX);
101+
}
102+
103+
public static void checkColumnNameBypassHiddenColumn(String columnName) throws AnalysisException {
97104
if (Strings.isNullOrEmpty(columnName) || !columnName.matches(getColumnNameRegex())) {
98105
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME,
99106
columnName, getColumnNameRegex());

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java‎

Lines changed: 79 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ public class ColumnDefinition {
6767
private int clusterKeyId = -1;
6868
private Optional<GeneratedColumnDesc> generatedColumnDesc = Optional.empty();
6969
private Set<String> generatedColumnsThatReferToThis = new HashSet<>();
70+
// if add hidden column, must set enableAddHiddenColumn true
71+
private boolean enableAddHiddenColumn = false;
7072

7173
public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType aggType, boolean isNullable,
7274
Optional<DefaultValue> defaultValue, String comment) {
@@ -286,7 +288,14 @@ private void checkKeyColumnType(boolean isOlap) {
286288
public void validate(boolean isOlap, Set<String> keysSet, Set<String> clusterKeySet, boolean isEnableMergeOnWrite,
287289
KeysType keysType) {
288290
try {
289-
FeNameFormat.checkColumnName(name);
291+
// if enableAddHiddenColumn is true, can add hidden column.
292+
// So does not check if the column name starts with __DORIS_
293+
if (enableAddHiddenColumn) {
294+
FeNameFormat.checkColumnNameBypassHiddenColumn(name);
295+
} else {
296+
FeNameFormat.checkColumnName(name);
297+
}
298+
290299
FeNameFormat.checkColumnCommentLength(comment);
291300
} catch (Exception e) {
292301
throw new AnalysisException(e.getMessage(), e);
@@ -523,43 +532,90 @@ public Column translateToCatalogStyleForSchemaChange() {
523532
return column;
524533
}
525534

526-
// hidden column
535+
/**
536+
* add hidden column
537+
*/
527538
public static ColumnDefinition newDeleteSignColumnDefinition() {
528-
return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, null, false,
529-
Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris delete flag hidden column", false);
539+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, null,
540+
false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
541+
"doris delete flag hidden column", false);
542+
columnDefinition.setEnableAddHiddenColumn(true);
543+
544+
return columnDefinition;
530545
}
531546

547+
/**
548+
* add hidden column
549+
*/
532550
public static ColumnDefinition newDeleteSignColumnDefinition(AggregateType aggregateType) {
533-
return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, aggregateType, false,
534-
Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris delete flag hidden column", false);
551+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false,
552+
aggregateType, false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
553+
"doris delete flag hidden column", false);
554+
columnDefinition.setEnableAddHiddenColumn(true);
555+
556+
return columnDefinition;
535557
}
536558

559+
/**
560+
* add hidden column
561+
*/
537562
public static ColumnDefinition newSequenceColumnDefinition(DataType type) {
538-
return new ColumnDefinition(Column.SEQUENCE_COL, type, false, null, true,
539-
Optional.empty(), "sequence column hidden column", false);
563+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.SEQUENCE_COL, type, false, null,
564+
true, Optional.empty(),
565+
"sequence column hidden column", false);
566+
columnDefinition.setEnableAddHiddenColumn(true);
567+
568+
return columnDefinition;
540569
}
541570

571+
/**
572+
* add hidden column
573+
*/
542574
public static ColumnDefinition newSequenceColumnDefinition(DataType type, AggregateType aggregateType) {
543-
return new ColumnDefinition(Column.SEQUENCE_COL, type, false, aggregateType, true,
544-
Optional.empty(), "sequence column hidden column", false);
575+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.SEQUENCE_COL, type, false, aggregateType,
576+
true, Optional.empty(),
577+
"sequence column hidden column", false);
578+
columnDefinition.setEnableAddHiddenColumn(true);
579+
580+
return columnDefinition;
545581
}
546582

583+
/**
584+
* add hidden column
585+
*/
547586
public static ColumnDefinition newRowStoreColumnDefinition(AggregateType aggregateType) {
548-
return new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false, aggregateType, false,
549-
Optional.of(new DefaultValue("")), "doris row store hidden column", false);
587+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false,
588+
aggregateType, false, Optional.of(new DefaultValue("")),
589+
"doris row store hidden column", false);
590+
columnDefinition.setEnableAddHiddenColumn(true);
591+
592+
return columnDefinition;
550593
}
551594

595+
/**
596+
* add hidden column
597+
*/
552598
public static ColumnDefinition newVersionColumnDefinition(AggregateType aggregateType) {
553-
return new ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE, false, aggregateType, false,
554-
Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris version hidden column", false);
599+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE, false,
600+
aggregateType, false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
601+
"doris version hidden column", false);
602+
columnDefinition.setEnableAddHiddenColumn(true);
603+
604+
return columnDefinition;
555605
}
556606

557-
// used in CreateTableInfo.validate(), specify the default value as DefaultValue.NULL_DEFAULT_VALUE
558-
// becasue ColumnDefinition.validate() will check that bitmap type column don't set default value
559-
// and then set the default value of that column to bitmap_empty()
607+
/**
608+
* used in CreateTableInfo.validate(), specify the default value as DefaultValue.NULL_DEFAULT_VALUE
609+
* becasue ColumnDefinition.validate() will check that bitmap type column don't set default value
610+
* and then set the default value of that column to bitmap_empty()
611+
*/
560612
public static ColumnDefinition newSkipBitmapColumnDef(AggregateType aggregateType) {
561-
return new ColumnDefinition(Column.SKIP_BITMAP_COL, BitmapType.INSTANCE, false, aggregateType, false,
562-
Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE), "doris skip bitmap hidden column", false);
613+
ColumnDefinition columnDefinition = new ColumnDefinition(Column.SKIP_BITMAP_COL, BitmapType.INSTANCE, false,
614+
aggregateType, false, Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE),
615+
"doris skip bitmap hidden column", false);
616+
columnDefinition.setEnableAddHiddenColumn(true);
617+
618+
return columnDefinition;
563619
}
564620

565621
public Optional<GeneratedColumnDesc> getGeneratedColumnDesc() {
@@ -574,6 +630,10 @@ public void addGeneratedColumnsThatReferToThis(List<String> list) {
574630
generatedColumnsThatReferToThis.addAll(list);
575631
}
576632

633+
public void setEnableAddHiddenColumn(boolean enableAddHiddenColumn) {
634+
this.enableAddHiddenColumn = enableAddHiddenColumn;
635+
}
636+
577637
private void validateGeneratedColumnInfo() {
578638
// for generated column
579639
if (generatedColumnDesc.isPresent()) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,9 @@ public void validate(ConnectContext ctx) throws UserException {
7272
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME,
7373
colName, FeNameFormat.getColumnNameRegex());
7474
}
75+
if (colName.startsWith(Column.HIDDEN_COLUMN_PREFIX)) {
76+
throw new AnalysisException("Do not support drop hidden column");
77+
}
7578

7679
TableIf table = Env.getCurrentEnv().getCatalogMgr()
7780
.getCatalogOrDdlException(tableName.getCtl())

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.apache.doris.alter.AlterOpType;
2121
import org.apache.doris.analysis.AlterTableClause;
2222
import org.apache.doris.analysis.ColumnRenameClause;
23+
import org.apache.doris.catalog.Column;
2324
import org.apache.doris.common.AnalysisException;
2425
import org.apache.doris.common.FeNameFormat;
2526
import org.apache.doris.common.UserException;
@@ -61,6 +62,10 @@ public void validate(ConnectContext ctx) throws UserException {
6162
throw new AnalysisException("New column name is not set");
6263
}
6364

65+
if (colName.startsWith(Column.HIDDEN_COLUMN_PREFIX)) {
66+
throw new AnalysisException("Do not support rename hidden column");
67+
}
68+
6469
FeNameFormat.checkColumnName(newColName);
6570
}
6671

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
19+
suite("test_alter_table_on_hidden_column", "schema_change") {
20+
def tableName = "alter_table_hidden_column"
21+
22+
sql "DROP TABLE IF EXISTS ${tableName}"
23+
sql """
24+
CREATE TABLE IF NOT EXISTS ${tableName} (
25+
k1 INT,
26+
value1 INT,
27+
value2 INT,
28+
value3 INT
29+
)
30+
DUPLICATE KEY (k1)
31+
DISTRIBUTED BY HASH(k1) BUCKETS 1
32+
properties(
33+
"replication_num" = "1"
34+
);
35+
"""
36+
37+
// rename hidden column
38+
try {
39+
sql "ALTER TABLE ${tableName} RENAME COLUMN __DORIS_VERSION_COL__ test;"
40+
} catch (Exception ex) {
41+
assert("${ex}".contains("Do not support rename hidden column"))
42+
}
43+
44+
// drop hidden column
45+
try {
46+
sql "ALTER TABLE ${tableName} DROP COLUMN __DORIS_VERSION_COL__;"
47+
} catch (Exception ex) {
48+
assert("${ex}".contains("Do not support drop hidden column"))
49+
}
50+
51+
// add a column name starting with __DORIS_
52+
try {
53+
sql "ALTER TABLE ${tableName} ADD COLUMN __DORIS_VERSION_COL__ bigint;"
54+
} catch (Exception ex) {
55+
assert("${ex}".contains("column name can't start with"))
56+
}
57+
58+
// add columns, a column name starting with __DORIS_
59+
try {
60+
sql "ALTER TABLE ${tableName} ADD COLUMN (c1 int, __DORIS_VERSION_COL__ bigint);"
61+
} catch (Exception ex) {
62+
assert("${ex}".contains("column name can't start with"))
63+
}
64+
65+
// modify hidden column comment
66+
try {
67+
sql "ALTER TABLE ${tableName} MODIFY COLUMN __DORIS_VERSION_COL__ COMMENT 'test';"
68+
} catch (Exception ex) {
69+
assert("${ex}".contains("Unknown column"))
70+
}
71+
72+
// modify hidden column type
73+
try {
74+
sql "ALTER TABLE ${tableName} MODIFY COLUMN __DORIS_VERSION_COL__ VARCHAR(64);"
75+
} catch (Exception ex) {
76+
assert("${ex}".contains("column name can't start with"))
77+
}
78+
79+
}

0 commit comments

Comments
 (0)