Skip to content

Commit cb68df6

Browse files
committed
[improvement](ddl) Preserve internal state type tables
### What problem does this PR solve? Issue Number: close apache#65865 Related PR: apache#66664 Problem Summary: Restricting HLL, QUANTILE_STATE, and AGG_STATE columns to Aggregate Key tables also rejected the internal partition statistics table, whose Unique Key schema contains an HLL column. The new negative regression suite could additionally race with compatibility suites because they temporarily mutate the same global FE config. Allow state types for FE internal sessions, cover creation of the internal statistics schema, preserve downstream validation expectations, and serialize suites that change the compatibility config. ### Release note None ### Check List (For Author) - Test: Unit Test / Build - `./run-fe-ut.sh --run org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinitionTest` - `./run-fe-ut.sh --run org.apache.doris.alter.InternalSchemaAlterTest,org.apache.doris.catalog.CreateTableWithBloomFilterIndexTest` - `./build.sh -j 192` - Regression tests were not run because no Doris cluster was deployed in this worktree - Behavior changed: No. This preserves required internal schemas while keeping the user-facing restriction unchanged - Does this need documentation: No
1 parent 6c42a13 commit cb68df6

18 files changed

Lines changed: 68 additions & 30 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -370,19 +370,24 @@ private void checkKeyColumnType(boolean isOlap) {
370370
*/
371371
public void validate(boolean isOlap, Set<String> keysSet, Set<String> clusterKeySet, boolean isEnableMergeOnWrite,
372372
KeysType keysType) {
373-
validateInternal(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType, false);
373+
validate(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType, false);
374+
}
375+
376+
public void validate(boolean isOlap, Set<String> keysSet, Set<String> clusterKeySet, boolean isEnableMergeOnWrite,
377+
KeysType keysType, boolean isInternalSession) {
378+
validateInternal(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType, false, isInternalSession);
374379
}
375380

376381
/**
377382
* Validate a nested field whose name is scoped by its parent path rather than the Doris top-level column namespace.
378383
*/
379384
public void validateNestedColumn(boolean isOlap, Set<String> keysSet, Set<String> clusterKeySet,
380385
boolean isEnableMergeOnWrite, KeysType keysType) {
381-
validateInternal(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType, true);
386+
validateInternal(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType, true, false);
382387
}
383388

384389
private void validateInternal(boolean isOlap, Set<String> keysSet, Set<String> clusterKeySet,
385-
boolean isEnableMergeOnWrite, KeysType keysType, boolean nestedColumn) {
390+
boolean isEnableMergeOnWrite, KeysType keysType, boolean nestedColumn, boolean isInternalSession) {
386391
try {
387392
// if enableAddHiddenColumn is true, can add hidden column.
388393
// So does not check if the column name starts with __DORIS_
@@ -400,7 +405,7 @@ private void validateInternal(boolean isOlap, Set<String> keysSet, Set<String> c
400405
}
401406
type.validateDataType();
402407
type = updateCharacterTypeLength(type);
403-
if (isOlap && keysType != KeysType.AGG_KEYS && isAggregateTableOnlyType()
408+
if (!isInternalSession && isOlap && keysType != KeysType.AGG_KEYS && isAggregateTableOnlyType()
404409
&& !Config.allow_non_aggregate_table_state_types) {
405410
throw new AnalysisException(String.format(
406411
"%s type is only supported in aggregate key tables, column: %s. "

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -803,8 +803,9 @@ public void validate(ConnectContext ctx) {
803803
keysSet.addAll(keys);
804804
Set<String> orderKeySet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
805805
orderKeySet.addAll(sortOrderFields.stream().map(SortFieldInfo::getColumnName).collect(Collectors.toSet()));
806+
boolean isInternalSession = ctx.getState().isInternal();
806807
columns.forEach(c -> c.validate(targetIsInternalCatalog, keysSet, orderKeySet, finalEnableMergeOnWrite,
807-
keysType));
808+
keysType, isInternalSession));
808809

809810
// validate index
810811
if (!indexes.isEmpty()) {

‎fe/fe-core/src/test/java/org/apache/doris/alter/InternalSchemaAlterTest.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.apache.doris.catalog.OlapTable;
2626
import org.apache.doris.catalog.Partition;
2727
import org.apache.doris.catalog.PartitionInfo;
28+
import org.apache.doris.catalog.PrimitiveType;
2829
import org.apache.doris.common.AnalysisException;
2930
import org.apache.doris.common.Config;
3031
import org.apache.doris.common.FeConstants;
@@ -89,4 +90,13 @@ public void testCheckAuditLogTable() throws AnalysisException {
8990
Assertions.assertNotNull(table.getColumn(def.getName()));
9091
}
9192
}
93+
94+
@Test
95+
public void testCheckPartitionStatisticsTable() throws AnalysisException {
96+
Database db = Env.getCurrentEnv().getCatalogMgr()
97+
.getInternalCatalog().getDbNullable(FeConstants.INTERNAL_DB_NAME);
98+
Assertions.assertNotNull(db);
99+
OlapTable table = db.getOlapTableOrAnalysisException(StatisticConstants.PARTITION_STATISTIC_TBL_NAME);
100+
Assertions.assertEquals(PrimitiveType.HLL, table.getColumn("ndv").getType().getPrimitiveType());
101+
}
92102
}

‎fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableWithBloomFilterIndexTest.java‎

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.doris.catalog;
1919

20+
import org.apache.doris.common.Config;
2021
import org.apache.doris.common.DdlException;
2122
import org.apache.doris.common.ExceptionChecker;
2223
import org.apache.doris.nereids.exceptions.AnalysisException;
@@ -404,18 +405,24 @@ public void testCreateTableWithJsonBloomFilterIndex() {
404405

405406
@Test
406407
public void testCreateTableWithHllBloomFilterIndex() {
407-
ExceptionChecker.expectThrowsWithMsg(DdlException.class,
408-
" HLL is not supported in bloom filter index. invalid column: k1",
409-
() -> createTable("CREATE TABLE test.tbl_hll_bf (\n"
410-
+ "v1 INT,\n"
411-
+ "k1 HLL\n"
412-
+ ") ENGINE=OLAP\n"
413-
+ "DUPLICATE KEY(v1)\n"
414-
+ "DISTRIBUTED BY HASH(v1) BUCKETS 1\n"
415-
+ "PROPERTIES (\n"
416-
+ "\"bloom_filter_columns\" = \"k1\",\n"
417-
+ "\"replication_num\" = \"1\"\n"
418-
+ ");"));
408+
boolean originalValue = Config.allow_non_aggregate_table_state_types;
409+
Config.allow_non_aggregate_table_state_types = true;
410+
try {
411+
ExceptionChecker.expectThrowsWithMsg(DdlException.class,
412+
" HLL is not supported in bloom filter index. invalid column: k1",
413+
() -> createTable("CREATE TABLE test.tbl_hll_bf (\n"
414+
+ "v1 INT,\n"
415+
+ "k1 HLL\n"
416+
+ ") ENGINE=OLAP\n"
417+
+ "DUPLICATE KEY(v1)\n"
418+
+ "DISTRIBUTED BY HASH(v1) BUCKETS 1\n"
419+
+ "PROPERTIES (\n"
420+
+ "\"bloom_filter_columns\" = \"k1\",\n"
421+
+ "\"replication_num\" = \"1\"\n"
422+
+ ");"));
423+
} finally {
424+
Config.allow_non_aggregate_table_state_types = originalValue;
425+
}
419426
}
420427

421428
@Test

‎fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,17 @@ public void testStateTypesRemainSupportedInAggregateKeyTable() {
110110
KeysType.AGG_KEYS));
111111
}
112112

113+
@Test
114+
public void testInternalSessionAllowsStateTypesInNonAggregateTable() {
115+
for (KeysType keysType : ImmutableList.of(KeysType.DUP_KEYS, KeysType.UNIQUE_KEYS)) {
116+
for (DataType type : aggregateTableOnlyTypes()) {
117+
ColumnDefinition column = new ColumnDefinition(
118+
"v", type, false, null, false, Optional.empty(), "");
119+
Assertions.assertDoesNotThrow(() -> validateInternalColumn(column, keysType));
120+
}
121+
}
122+
}
123+
113124
private static ImmutableList<DataType> aggregateTableOnlyTypes() {
114125
return ImmutableList.of(HllType.INSTANCE, QuantileStateType.INSTANCE, aggStateType());
115126
}
@@ -121,4 +132,8 @@ private static AggStateType aggStateType() {
121132
private static void validateColumn(ColumnDefinition column, KeysType keysType) {
122133
column.validate(true, ImmutableSet.of("k"), ImmutableSet.of(), true, keysType);
123134
}
135+
136+
private static void validateInternalColumn(ColumnDefinition column, KeysType keysType) {
137+
column.validate(true, ImmutableSet.of("k"), ImmutableSet.of(), true, keysType, true);
138+
}
124139
}

‎regression-test/suites/correctness_p0/test_default_hll.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
suite("test_default_hll") {
18+
suite("test_default_hll", "nonConcurrent") {
1919
setFeConfigTemporary([allow_non_aggregate_table_state_types: true]) {
2020
def tableName = "test_default_hll"
2121

‎regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_hll.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
suite("test_duplicate_table_hll") {
18+
suite("test_duplicate_table_hll", "nonConcurrent") {
1919
setFeConfigTemporary([allow_non_aggregate_table_state_types: true]) {
2020

2121
sql "sync;"

‎regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_quantile_state.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
suite("test_duplicate_table_quantile_state") {
18+
suite("test_duplicate_table_quantile_state", "nonConcurrent") {
1919
setFeConfigTemporary([allow_non_aggregate_table_state_types: true]) {
2020

2121
sql "sync;"

‎regression-test/suites/data_model_p0/test_state_types_only_in_aggregate_table.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
suite("test_state_types_only_in_aggregate_table") {
18+
suite("test_state_types_only_in_aggregate_table", "nonConcurrent") {
1919
sql "set enable_agg_state=true"
2020

2121
sql "drop table if exists state_type_dup_hll"

‎regression-test/suites/data_model_p0/unique/test_unique_hll.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
suite("test_unique_table_hll") {
18+
suite("test_unique_table_hll", "nonConcurrent") {
1919
setFeConfigTemporary([allow_non_aggregate_table_state_types: true]) {
2020

2121
for (def enable_mow : [true, false]) {

0 commit comments

Comments
 (0)