Skip to content

Commit 42f8f2e

Browse files
authored
[improvement](ddl) Restrict state types to aggregate tables (#66963)
### What problem does this PR solve? Issue Number: close #65865 Related PR: #66664 Problem Summary: HLL, QUANTILE_STATE, and AGG_STATE columns in non-aggregate table models can enter row conversion paths that do not support nullable aggregate-state values and fail at runtime. Reject new definitions of these types in Duplicate Key and Unique Key tables by default, while retaining a temporary FE compatibility switch for migration. Existing Aggregate Key tables remain supported. ### Release note HLL, QUANTILE_STATE, and AGG_STATE columns are restricted to Aggregate Key tables by default. The temporary FE config `enable_non_aggregate_table_state_types=true` can restore the previous definition behavior during migration and will be removed after the compatibility transition period. ### Check List (For Author) - Test: Unit Test / Build - `./run-fe-ut.sh --run org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinitionTest` - `./build.sh --fe` - `./build.sh --be` - Regression test added but not run because an isolated cluster rebuild requires separate destructive-action approval - Behavior changed: Yes. New HLL, QUANTILE_STATE, and AGG_STATE columns are rejected in non-Aggregate Key tables by default - Does this need documentation: No. The compatibility switch is temporary and carries its removal notice in the FE config description
1 parent 0ae0115 commit 42f8f2e

21 files changed

Lines changed: 355 additions & 19 deletions

File tree

‎fe/fe-common/src/main/java/org/apache/doris/common/Config.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1859,6 +1859,11 @@ public class Config extends ConfigBase {
18591859
@ConfField(mutable = true, masterOnly = true)
18601860
public static boolean enable_quantile_state_type = true;
18611861

1862+
@ConfField(mutable = true, masterOnly = true, description = "Temporary compatibility switch that allows HLL, "
1863+
+ "QUANTILE_STATE, and AGG_STATE columns in non-aggregate key tables. Disabled by default. This switch "
1864+
+ "is intended only for migration and will be removed after the compatibility transition period.")
1865+
public static boolean enable_non_aggregate_table_state_types = false;
1866+
18621867
/*---------------------- JOB CONFIG START------------------------*/
18631868
/**
18641869
* The number of threads used to dispatch timer job.

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -698,7 +698,7 @@ private static void validateColumns(List<ColumnDefinition> columns, Set<String>
698698
if (col.getType().isVarBinaryType()) {
699699
throw new AnalysisException("MTMV do not support varbinary type : " + col.getName());
700700
}
701-
col.validate(true, keysSet, Sets.newHashSet(), finalEnableMergeOnWrite, keysType);
701+
col.validate(true, keysSet, Sets.newHashSet(), finalEnableMergeOnWrite, keysType, true);
702702
}
703703
}
704704

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.doris.catalog.Column;
2525
import org.apache.doris.catalog.KeysType;
2626
import org.apache.doris.common.CaseSensibility;
27+
import org.apache.doris.common.Config;
2728
import org.apache.doris.common.FeNameFormat;
2829
import org.apache.doris.common.util.SqlUtils;
2930
import org.apache.doris.nereids.exceptions.AnalysisException;
@@ -311,6 +312,10 @@ private String toSql(String columnNameSql, boolean includeComment) {
311312
return sb.toString();
312313
}
313314

315+
private boolean isAggregateTableOnlyType() {
316+
return type.isHllType() || type.isQuantileStateType() || type.isAggStateType();
317+
}
318+
314319
private DataType updateCharacterTypeLength(DataType dataType) {
315320
if (dataType instanceof ArrayType) {
316321
return ArrayType.of(updateCharacterTypeLength(((ArrayType) dataType).getItemType()));
@@ -384,19 +389,25 @@ private void checkKeyColumnType(boolean isOlap) {
384389
*/
385390
public void validate(boolean isOlap, Set<String> keysSet, Set<String> clusterKeySet, boolean isEnableMergeOnWrite,
386391
KeysType keysType) {
387-
validateInternal(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType, false);
392+
validate(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType, false);
393+
}
394+
395+
public void validate(boolean isOlap, Set<String> keysSet, Set<String> clusterKeySet, boolean isEnableMergeOnWrite,
396+
KeysType keysType, boolean isSystemGeneratedTable) {
397+
validateInternal(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType, false,
398+
isSystemGeneratedTable);
388399
}
389400

390401
/**
391402
* Validate a nested field whose name is scoped by its parent path rather than the Doris top-level column namespace.
392403
*/
393404
public void validateNestedColumn(boolean isOlap, Set<String> keysSet, Set<String> clusterKeySet,
394405
boolean isEnableMergeOnWrite, KeysType keysType) {
395-
validateInternal(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType, true);
406+
validateInternal(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType, true, false);
396407
}
397408

398409
private void validateInternal(boolean isOlap, Set<String> keysSet, Set<String> clusterKeySet,
399-
boolean isEnableMergeOnWrite, KeysType keysType, boolean nestedColumn) {
410+
boolean isEnableMergeOnWrite, KeysType keysType, boolean nestedColumn, boolean isSystemGeneratedTable) {
400411
try {
401412
// if enableAddHiddenColumn is true, can add hidden column.
402413
// So does not check if the column name starts with __DORIS_
@@ -414,6 +425,13 @@ private void validateInternal(boolean isOlap, Set<String> keysSet, Set<String> c
414425
}
415426
type.validateDataType();
416427
type = updateCharacterTypeLength(type);
428+
if (!isSystemGeneratedTable && isOlap && keysType != KeysType.AGG_KEYS && isAggregateTableOnlyType()
429+
&& !Config.enable_non_aggregate_table_state_types) {
430+
throw new AnalysisException(String.format(
431+
"%s type is only supported in aggregate key tables, column: %s. "
432+
+ "Set FE config 'enable_non_aggregate_table_state_types' to true to temporarily allow it",
433+
type.toSql(), name));
434+
}
417435
if (type.isArrayType()) {
418436
int depth = 0;
419437
DataType curType = type;

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -726,8 +726,10 @@ public void validate(ConnectContext ctx) {
726726
keysSet.addAll(keys);
727727
Set<String> orderKeySet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
728728
orderKeySet.addAll(sortOrderFields.stream().map(SortFieldInfo::getColumnName).collect(Collectors.toSet()));
729+
// Internal statistics tables need state columns. The internal-query flag can also be set by user SHOWs.
730+
boolean isSystemGeneratedTable = targetIsInternalCatalog && FeConstants.INTERNAL_DB_NAME.equals(dbName);
729731
columns.forEach(c -> c.validate(targetIsInternalCatalog, keysSet, orderKeySet, finalEnableMergeOnWrite,
730-
keysType));
732+
keysType, isSystemGeneratedTable));
731733

732734
try {
733735
invertedIndexFileStorageFormat =

‎fe/fe-core/src/test/java/org/apache/doris/alter/InternalSchemaAlterTest.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.apache.doris.catalog.OlapTable;
2626
import org.apache.doris.catalog.Partition;
2727
import org.apache.doris.catalog.PartitionInfo;
28+
import org.apache.doris.catalog.PrimitiveType;
2829
import org.apache.doris.common.AnalysisException;
2930
import org.apache.doris.common.Config;
3031
import org.apache.doris.common.FeConstants;
@@ -89,4 +90,13 @@ public void testCheckAuditLogTable() throws AnalysisException {
8990
Assertions.assertNotNull(table.getColumn(def.getName()));
9091
}
9192
}
93+
94+
@Test
95+
public void testCheckPartitionStatisticsTable() throws AnalysisException {
96+
Database db = Env.getCurrentEnv().getCatalogMgr()
97+
.getInternalCatalog().getDbNullable(FeConstants.INTERNAL_DB_NAME);
98+
Assertions.assertNotNull(db);
99+
OlapTable table = db.getOlapTableOrAnalysisException(StatisticConstants.PARTITION_STATISTIC_TBL_NAME);
100+
Assertions.assertEquals(PrimitiveType.HLL, table.getColumn("ndv").getType().getPrimitiveType());
101+
}
92102
}

‎fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,34 @@ protected void runBeforeAll() throws Exception {
5252
createDatabase("test");
5353
}
5454

55+
@Test
56+
public void testInternalQueryStateDoesNotExemptUserTable() {
57+
boolean originalAllowStateTypes = Config.enable_non_aggregate_table_state_types;
58+
boolean originalInternal = connectContext.getState().isInternal();
59+
boolean originalEnableAggState = connectContext.getSessionVariable().enableAggState;
60+
Config.enable_non_aggregate_table_state_types = false;
61+
connectContext.getSessionVariable().enableAggState = true;
62+
// An ordinary SHOW can leave the internal-query flag set on a user connection.
63+
connectContext.getState().setInternal(true);
64+
try {
65+
for (String keysType : new String[] {"DUPLICATE", "UNIQUE"}) {
66+
for (String type : new String[] {"HLL NOT NULL", "QUANTILE_STATE NOT NULL",
67+
"AGG_STATE<sum(INT NOT NULL)>"}) {
68+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
69+
() -> createTable("CREATE TABLE test.user_state_type (k INT, v " + type + ") "
70+
+ keysType + " KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 "
71+
+ "PROPERTIES('replication_num'='1')"));
72+
Assertions.assertTrue(exception.getMessage().contains(
73+
"type is only supported in aggregate key tables"));
74+
}
75+
}
76+
} finally {
77+
Config.enable_non_aggregate_table_state_types = originalAllowStateTypes;
78+
connectContext.getState().setInternal(originalInternal);
79+
connectContext.getSessionVariable().enableAggState = originalEnableAggState;
80+
}
81+
}
82+
5583
@Test
5684
public void testDuplicateCreateTable() throws Exception {
5785
// test

‎fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableWithBloomFilterIndexTest.java‎

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
import org.apache.doris.alter.AlterJobV2;
2121
import org.apache.doris.catalog.info.IndexType;
22+
import org.apache.doris.common.Config;
2223
import org.apache.doris.common.DdlException;
2324
import org.apache.doris.common.ExceptionChecker;
2425
import org.apache.doris.common.FeConstants;
@@ -446,18 +447,24 @@ public void testCreateTableWithJsonBloomFilterIndex() {
446447

447448
@Test
448449
public void testCreateTableWithHllBloomFilterIndex() {
449-
ExceptionChecker.expectThrowsWithMsg(DdlException.class,
450-
" HLL is not supported in bloom filter index. invalid column: k1",
451-
() -> createTable("CREATE TABLE test.tbl_hll_bf (\n"
452-
+ "v1 INT,\n"
453-
+ "k1 HLL\n"
454-
+ ") ENGINE=OLAP\n"
455-
+ "DUPLICATE KEY(v1)\n"
456-
+ "DISTRIBUTED BY HASH(v1) BUCKETS 1\n"
457-
+ "PROPERTIES (\n"
458-
+ "\"bloom_filter_columns\" = \"k1\",\n"
459-
+ "\"replication_num\" = \"1\"\n"
460-
+ ");"));
450+
boolean originalValue = Config.enable_non_aggregate_table_state_types;
451+
Config.enable_non_aggregate_table_state_types = true;
452+
try {
453+
ExceptionChecker.expectThrowsWithMsg(DdlException.class,
454+
" HLL is not supported in bloom filter index. invalid column: k1",
455+
() -> createTable("CREATE TABLE test.tbl_hll_bf (\n"
456+
+ "v1 INT,\n"
457+
+ "k1 HLL\n"
458+
+ ") ENGINE=OLAP\n"
459+
+ "DUPLICATE KEY(v1)\n"
460+
+ "DISTRIBUTED BY HASH(v1) BUCKETS 1\n"
461+
+ "PROPERTIES (\n"
462+
+ "\"bloom_filter_columns\" = \"k1\",\n"
463+
+ "\"replication_num\" = \"1\"\n"
464+
+ ");"));
465+
} finally {
466+
Config.enable_non_aggregate_table_state_types = originalValue;
467+
}
461468
}
462469

463470
@Test

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -352,6 +352,22 @@ public void testAnalyzeQueryNormal() throws Exception {
352352
Assertions.assertTrue(mtmvAnalyzeQueryInfo.getColumnDefinitions().size() == 2);
353353
}
354354

355+
@Test
356+
public void testCreateMTMVWithAggStateColumn() throws Exception {
357+
boolean originalEnableAggState = connectContext.getSessionVariable().enableAggState;
358+
connectContext.getSessionVariable().enableAggState = true;
359+
connectContext.setThreadLocalInfo();
360+
try {
361+
Assertions.assertDoesNotThrow(() -> createMvByNereids(
362+
"create materialized view mv_with_agg_state BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n"
363+
+ "DISTRIBUTED BY RANDOM BUCKETS 1\n"
364+
+ "PROPERTIES ('replication_num' = '1')\n"
365+
+ "as select id, sum_union(sum_state(score)) from test.T1 group by id"));
366+
} finally {
367+
connectContext.getSessionVariable().enableAggState = originalEnableAggState;
368+
}
369+
}
370+
355371
@Test
356372
public void testEnsureMTMVQueryUsable() throws Exception {
357373
createMvByNereids("create materialized view mv1 BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n"

‎fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,38 @@
1717

1818
package org.apache.doris.nereids.trees.plans.commands.info;
1919

20+
import org.apache.doris.catalog.AggregateType;
21+
import org.apache.doris.catalog.KeysType;
22+
import org.apache.doris.common.Config;
23+
import org.apache.doris.nereids.exceptions.AnalysisException;
24+
import org.apache.doris.nereids.types.AggStateType;
25+
import org.apache.doris.nereids.types.DataType;
26+
import org.apache.doris.nereids.types.HllType;
27+
import org.apache.doris.nereids.types.IntegerType;
28+
import org.apache.doris.nereids.types.QuantileStateType;
2029
import org.apache.doris.nereids.types.StringType;
2130

31+
import com.google.common.collect.ImmutableList;
32+
import com.google.common.collect.ImmutableSet;
33+
import org.junit.jupiter.api.AfterEach;
2234
import org.junit.jupiter.api.Assertions;
35+
import org.junit.jupiter.api.BeforeEach;
2336
import org.junit.jupiter.api.Test;
2437

38+
import java.util.Optional;
39+
2540
public class ColumnDefinitionTest {
2641

42+
@BeforeEach
43+
public void setUp() {
44+
Config.enable_non_aggregate_table_state_types = false;
45+
}
46+
47+
@AfterEach
48+
public void tearDown() {
49+
Config.enable_non_aggregate_table_state_types = false;
50+
}
51+
2752
@Test
2853
public void testNameEquals() {
2954
ColumnDefinition columnDefinition = new ColumnDefinition("col1", null, false, null, false, null, null);
@@ -43,4 +68,72 @@ public void testToSqlHandlesNullComment() {
4368
String sql = columnDefinition.toSql();
4469
Assertions.assertTrue(sql.endsWith("COMMENT \"\""));
4570
}
71+
72+
@Test
73+
public void testStateTypesRequireAggregateKeyTableByDefault() {
74+
for (KeysType keysType : ImmutableList.of(KeysType.DUP_KEYS, KeysType.UNIQUE_KEYS)) {
75+
for (DataType type : aggregateTableOnlyTypes()) {
76+
ColumnDefinition column = new ColumnDefinition(
77+
"v", type, false, null, false, Optional.empty(), "");
78+
79+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
80+
() -> validateColumn(column, keysType));
81+
Assertions.assertTrue(exception.getMessage().contains(
82+
type.toSql() + " type is only supported in aggregate key tables"));
83+
}
84+
}
85+
}
86+
87+
@Test
88+
public void testTemporaryConfigAllowsStateTypesInNonAggregateTable() {
89+
Config.enable_non_aggregate_table_state_types = true;
90+
91+
for (KeysType keysType : ImmutableList.of(KeysType.DUP_KEYS, KeysType.UNIQUE_KEYS)) {
92+
for (DataType type : aggregateTableOnlyTypes()) {
93+
ColumnDefinition column = new ColumnDefinition(
94+
"v", type, false, null, false, Optional.empty(), "");
95+
Assertions.assertDoesNotThrow(() -> validateColumn(column, keysType));
96+
}
97+
}
98+
}
99+
100+
@Test
101+
public void testStateTypesRemainSupportedInAggregateKeyTable() {
102+
Assertions.assertDoesNotThrow(() -> validateColumn(new ColumnDefinition(
103+
"v", HllType.INSTANCE, false, AggregateType.HLL_UNION, false, Optional.empty(), ""),
104+
KeysType.AGG_KEYS));
105+
Assertions.assertDoesNotThrow(() -> validateColumn(new ColumnDefinition(
106+
"v", QuantileStateType.INSTANCE, false, AggregateType.QUANTILE_UNION, false, Optional.empty(), ""),
107+
KeysType.AGG_KEYS));
108+
Assertions.assertDoesNotThrow(() -> validateColumn(new ColumnDefinition(
109+
"v", aggStateType(), false, AggregateType.GENERIC, false, Optional.empty(), ""),
110+
KeysType.AGG_KEYS));
111+
}
112+
113+
@Test
114+
public void testSystemGeneratedTableAllowsStateTypesInNonAggregateTable() {
115+
for (KeysType keysType : ImmutableList.of(KeysType.DUP_KEYS, KeysType.UNIQUE_KEYS)) {
116+
for (DataType type : aggregateTableOnlyTypes()) {
117+
ColumnDefinition column = new ColumnDefinition(
118+
"v", type, false, null, false, Optional.empty(), "");
119+
Assertions.assertDoesNotThrow(() -> validateSystemGeneratedColumn(column, keysType));
120+
}
121+
}
122+
}
123+
124+
private static ImmutableList<DataType> aggregateTableOnlyTypes() {
125+
return ImmutableList.of(HllType.INSTANCE, QuantileStateType.INSTANCE, aggStateType());
126+
}
127+
128+
private static AggStateType aggStateType() {
129+
return new AggStateType("sum", ImmutableList.of(IntegerType.INSTANCE), ImmutableList.of(false), false);
130+
}
131+
132+
private static void validateColumn(ColumnDefinition column, KeysType keysType) {
133+
column.validate(true, ImmutableSet.of("k"), ImmutableSet.of(), true, keysType);
134+
}
135+
136+
private static void validateSystemGeneratedColumn(ColumnDefinition column, KeysType keysType) {
137+
column.validate(true, ImmutableSet.of("k"), ImmutableSet.of(), true, keysType, true);
138+
}
46139
}

‎regression-test/suites/correctness_p0/test_default_hll.groovy‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
// under the License.
1717

1818
suite("test_default_hll") {
19+
withGlobalLock("enable_non_aggregate_table_state_types") {
20+
setFeConfigTemporary([enable_non_aggregate_table_state_types: true]) {
1921
def tableName = "test_default_hll"
2022

2123
sql """ DROP TABLE IF EXISTS ${tableName} """
@@ -96,4 +98,6 @@ suite("test_default_hll") {
9698

9799
qt_stream_load_csv1 """ select HLL_CARDINALITY(h1) from ${tableName} order by k; """
98100

99-
}
101+
}
102+
}
103+
}

0 commit comments

Comments
 (0)