Skip to content

Commit dba31be

Browse files
committed
[fix](ddl) Restrict state types to aggregate tables
### What problem does this PR solve? Issue Number: close apache#65865 Related PR: apache#66664 Problem Summary: HLL, QUANTILE_STATE, and AGG_STATE columns in non-aggregate table models can enter row conversion paths that do not support nullable aggregate-state values and fail at runtime. Reject new definitions of these types in Duplicate Key and Unique Key tables by default, while retaining a temporary FE compatibility switch for migration. Existing Aggregate Key tables remain supported. ### Release note HLL, QUANTILE_STATE, and AGG_STATE columns are restricted to Aggregate Key tables by default. The temporary FE config `allow_non_aggregate_table_state_types=true` can restore the previous definition behavior during migration and will be removed after the compatibility transition period. ### Check List (For Author) - Test: Unit Test / Build - `./run-fe-ut.sh --run org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinitionTest` - `./build.sh --fe` - `./build.sh --be` - Regression test added but not run because an isolated cluster rebuild requires separate destructive-action approval - Behavior changed: Yes. New HLL, QUANTILE_STATE, and AGG_STATE columns are rejected in non-Aggregate Key tables by default - Does this need documentation: No. The compatibility switch is temporary and carries its removal notice in the FE config description
1 parent 190eb6a commit dba31be

16 files changed

Lines changed: 227 additions & 3 deletions

File tree

‎fe/fe-common/src/main/java/org/apache/doris/common/Config.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1837,6 +1837,11 @@ public class Config extends ConfigBase {
18371837
@ConfField(mutable = true, masterOnly = true)
18381838
public static boolean enable_quantile_state_type = true;
18391839

1840+
@ConfField(mutable = true, masterOnly = true, description = "Temporary compatibility switch that allows HLL, "
1841+
+ "QUANTILE_STATE, and AGG_STATE columns in non-aggregate key tables. Disabled by default. This switch "
1842+
+ "is intended only for migration and will be removed after the compatibility transition period.")
1843+
public static boolean allow_non_aggregate_table_state_types = false;
1844+
18401845
/*---------------------- JOB CONFIG START------------------------*/
18411846
/**
18421847
* The number of threads used to dispatch timer job.

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.doris.catalog.Column;
2525
import org.apache.doris.catalog.KeysType;
2626
import org.apache.doris.common.CaseSensibility;
27+
import org.apache.doris.common.Config;
2728
import org.apache.doris.common.FeNameFormat;
2829
import org.apache.doris.common.util.SqlUtils;
2930
import org.apache.doris.nereids.exceptions.AnalysisException;
@@ -311,6 +312,10 @@ private String toSql(String columnNameSql, boolean includeComment) {
311312
return sb.toString();
312313
}
313314

315+
private boolean isAggregateTableOnlyType() {
316+
return type.isHllType() || type.isQuantileStateType() || type.isAggStateType();
317+
}
318+
314319
private DataType updateCharacterTypeLength(DataType dataType) {
315320
if (dataType instanceof ArrayType) {
316321
return ArrayType.of(updateCharacterTypeLength(((ArrayType) dataType).getItemType()));
@@ -395,6 +400,13 @@ private void validateInternal(boolean isOlap, Set<String> keysSet, Set<String> c
395400
}
396401
type.validateDataType();
397402
type = updateCharacterTypeLength(type);
403+
if (isOlap && keysType != KeysType.AGG_KEYS && isAggregateTableOnlyType()
404+
&& !Config.allow_non_aggregate_table_state_types) {
405+
throw new AnalysisException(String.format(
406+
"%s type is only supported in aggregate key tables, column: %s. "
407+
+ "Set FE config 'allow_non_aggregate_table_state_types' to true to temporarily allow it",
408+
type.toSql(), name));
409+
}
398410
if (type.isArrayType()) {
399411
int depth = 0;
400412
DataType curType = type;

‎fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,38 @@
1717

1818
package org.apache.doris.nereids.trees.plans.commands.info;
1919

20+
import org.apache.doris.catalog.AggregateType;
21+
import org.apache.doris.catalog.KeysType;
22+
import org.apache.doris.common.Config;
23+
import org.apache.doris.nereids.exceptions.AnalysisException;
24+
import org.apache.doris.nereids.types.AggStateType;
25+
import org.apache.doris.nereids.types.DataType;
26+
import org.apache.doris.nereids.types.HllType;
27+
import org.apache.doris.nereids.types.IntegerType;
28+
import org.apache.doris.nereids.types.QuantileStateType;
2029
import org.apache.doris.nereids.types.StringType;
2130

31+
import com.google.common.collect.ImmutableList;
32+
import com.google.common.collect.ImmutableSet;
33+
import org.junit.jupiter.api.AfterEach;
2234
import org.junit.jupiter.api.Assertions;
35+
import org.junit.jupiter.api.BeforeEach;
2336
import org.junit.jupiter.api.Test;
2437

38+
import java.util.Optional;
39+
2540
public class ColumnDefinitionTest {
2641

42+
@BeforeEach
43+
public void setUp() {
44+
Config.allow_non_aggregate_table_state_types = false;
45+
}
46+
47+
@AfterEach
48+
public void tearDown() {
49+
Config.allow_non_aggregate_table_state_types = false;
50+
}
51+
2752
@Test
2853
public void testNameEquals() {
2954
ColumnDefinition columnDefinition = new ColumnDefinition("col1", null, false, null, false, null, null);
@@ -43,4 +68,57 @@ public void testToSqlHandlesNullComment() {
4368
String sql = columnDefinition.toSql();
4469
Assertions.assertTrue(sql.endsWith("COMMENT \"\""));
4570
}
71+
72+
@Test
73+
public void testStateTypesRequireAggregateKeyTableByDefault() {
74+
for (KeysType keysType : ImmutableList.of(KeysType.DUP_KEYS, KeysType.UNIQUE_KEYS)) {
75+
for (DataType type : aggregateTableOnlyTypes()) {
76+
ColumnDefinition column = new ColumnDefinition(
77+
"v", type, false, null, false, Optional.empty(), "");
78+
79+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
80+
() -> validateColumn(column, keysType));
81+
Assertions.assertTrue(exception.getMessage().contains(
82+
type.toSql() + " type is only supported in aggregate key tables"));
83+
}
84+
}
85+
}
86+
87+
@Test
88+
public void testTemporaryConfigAllowsStateTypesInNonAggregateTable() {
89+
Config.allow_non_aggregate_table_state_types = true;
90+
91+
for (KeysType keysType : ImmutableList.of(KeysType.DUP_KEYS, KeysType.UNIQUE_KEYS)) {
92+
for (DataType type : aggregateTableOnlyTypes()) {
93+
ColumnDefinition column = new ColumnDefinition(
94+
"v", type, false, null, false, Optional.empty(), "");
95+
Assertions.assertDoesNotThrow(() -> validateColumn(column, keysType));
96+
}
97+
}
98+
}
99+
100+
@Test
101+
public void testStateTypesRemainSupportedInAggregateKeyTable() {
102+
Assertions.assertDoesNotThrow(() -> validateColumn(new ColumnDefinition(
103+
"v", HllType.INSTANCE, false, AggregateType.HLL_UNION, false, Optional.empty(), ""),
104+
KeysType.AGG_KEYS));
105+
Assertions.assertDoesNotThrow(() -> validateColumn(new ColumnDefinition(
106+
"v", QuantileStateType.INSTANCE, false, AggregateType.QUANTILE_UNION, false, Optional.empty(), ""),
107+
KeysType.AGG_KEYS));
108+
Assertions.assertDoesNotThrow(() -> validateColumn(new ColumnDefinition(
109+
"v", aggStateType(), false, AggregateType.GENERIC, false, Optional.empty(), ""),
110+
KeysType.AGG_KEYS));
111+
}
112+
113+
private static ImmutableList<DataType> aggregateTableOnlyTypes() {
114+
return ImmutableList.of(HllType.INSTANCE, QuantileStateType.INSTANCE, aggStateType());
115+
}
116+
117+
private static AggStateType aggStateType() {
118+
return new AggStateType("sum", ImmutableList.of(IntegerType.INSTANCE), ImmutableList.of(false), false);
119+
}
120+
121+
private static void validateColumn(ColumnDefinition column, KeysType keysType) {
122+
column.validate(true, ImmutableSet.of("k"), ImmutableSet.of(), true, keysType);
123+
}
46124
}

‎regression-test/suites/correctness_p0/test_default_hll.groovy‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
// under the License.
1717

1818
suite("test_default_hll") {
19+
setFeConfigTemporary([allow_non_aggregate_table_state_types: true]) {
1920
def tableName = "test_default_hll"
2021

2122
sql """ DROP TABLE IF EXISTS ${tableName} """
@@ -96,4 +97,5 @@ suite("test_default_hll") {
9697

9798
qt_stream_load_csv1 """ select HLL_CARDINALITY(h1) from ${tableName} order by k; """
9899

99-
}
100+
}
101+
}

‎regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_hll.groovy‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
// under the License.
1717

1818
suite("test_duplicate_table_hll") {
19+
setFeConfigTemporary([allow_non_aggregate_table_state_types: true]) {
1920

2021
sql "sync;"
2122

@@ -68,4 +69,5 @@ suite("test_duplicate_table_hll") {
6869
DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1"); """
6970
exception "Key column can not set complex type:k"
7071
}
72+
}
7173
}

‎regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_quantile_state.groovy‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
// under the License.
1717

1818
suite("test_duplicate_table_quantile_state") {
19+
setFeConfigTemporary([allow_non_aggregate_table_state_types: true]) {
1920

2021
sql "sync;"
2122

@@ -64,4 +65,5 @@ suite("test_duplicate_table_quantile_state") {
6465
DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1"); """
6566
exception "Key column can not set complex type:k"
6667
}
68+
}
6769
}
Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
suite("test_state_types_only_in_aggregate_table") {
19+
sql "set enable_agg_state=true"
20+
21+
sql "drop table if exists state_type_dup_hll"
22+
test {
23+
sql """
24+
create table state_type_dup_hll (
25+
k int,
26+
v hll not null
27+
) duplicate key(k)
28+
distributed by hash(k) buckets 1
29+
properties("replication_num" = "1")
30+
"""
31+
exception "type is only supported in aggregate key tables"
32+
}
33+
34+
sql "drop table if exists state_type_unique_quantile"
35+
test {
36+
sql """
37+
create table state_type_unique_quantile (
38+
k int,
39+
v quantile_state not null
40+
) unique key(k)
41+
distributed by hash(k) buckets 1
42+
properties("replication_num" = "1")
43+
"""
44+
exception "type is only supported in aggregate key tables"
45+
}
46+
47+
sql "drop table if exists state_type_dup_agg_state"
48+
test {
49+
sql """
50+
create table state_type_dup_agg_state (
51+
k int,
52+
v agg_state<sum(int not null)> generic
53+
) duplicate key(k)
54+
distributed by hash(k) buckets 1
55+
properties("replication_num" = "1")
56+
"""
57+
exception "type is only supported in aggregate key tables"
58+
}
59+
60+
sql "drop table if exists state_type_alter_dup"
61+
sql """
62+
create table state_type_alter_dup (
63+
k int,
64+
v int
65+
) duplicate key(k)
66+
distributed by hash(k) buckets 1
67+
properties("replication_num" = "1")
68+
"""
69+
test {
70+
sql "alter table state_type_alter_dup add column h hll not null"
71+
exception "type is only supported in aggregate key tables"
72+
}
73+
test {
74+
sql "alter table state_type_alter_dup add column q quantile_state not null"
75+
exception "type is only supported in aggregate key tables"
76+
}
77+
test {
78+
sql "alter table state_type_alter_dup add column a agg_state<sum(int not null)> generic"
79+
exception "type is only supported in aggregate key tables"
80+
}
81+
82+
sql "drop table if exists state_type_aggregate"
83+
sql """
84+
create table state_type_aggregate (
85+
k int,
86+
h hll hll_union not null,
87+
q quantile_state quantile_union not null,
88+
a agg_state<sum(int not null)> generic
89+
) aggregate key(k)
90+
distributed by hash(k) buckets 1
91+
properties("replication_num" = "1")
92+
"""
93+
94+
setFeConfigTemporary([allow_non_aggregate_table_state_types: true]) {
95+
sql "drop table if exists state_type_compatibility_dup"
96+
sql """
97+
create table state_type_compatibility_dup (
98+
k int,
99+
h hll not null,
100+
q quantile_state not null,
101+
a agg_state<sum(int not null)> generic
102+
) duplicate key(k)
103+
distributed by hash(k) buckets 1
104+
properties("replication_num" = "1")
105+
"""
106+
}
107+
}

‎regression-test/suites/data_model_p0/unique/test_unique_hll.groovy‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
// under the License.
1717

1818
suite("test_unique_table_hll") {
19+
setFeConfigTemporary([allow_non_aggregate_table_state_types: true]) {
1920

2021
for (def enable_mow : [true, false]) {
2122
sql "sync;"
@@ -70,4 +71,5 @@ suite("test_unique_table_hll") {
7071
exception "Key column can not set complex type:k"
7172
}
7273
}
74+
}
7375
}

‎regression-test/suites/data_model_p0/unique/test_unique_quantile_state.groovy‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
// under the License.
1717

1818
suite("test_unique_table_quantile_state") {
19+
setFeConfigTemporary([allow_non_aggregate_table_state_types: true]) {
1920

2021
for (def enable_mow : [true, false]) {
2122
sql "sync;"
@@ -66,4 +67,5 @@ suite("test_unique_table_quantile_state") {
6667
exception "Key column can not set complex type:k"
6768
}
6869
}
70+
}
6971
}

‎regression-test/suites/external_table_p0/remote_doris/test_remote_doris_unique_table_select.groovy‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
// under the License.
1717

1818
suite("test_remote_doris_unique_table_select", "p0,external") {
19+
setFeConfigTemporary([allow_non_aggregate_table_state_types: true]) {
1920
String remote_doris_host = context.config.otherConfigs.get("extArrowFlightSqlHost")
2021
String remote_doris_arrow_port = context.config.otherConfigs.get("extArrowFlightSqlPort")
2122
String remote_doris_http_port = context.config.otherConfigs.get("extArrowFlightHttpPort")
@@ -235,4 +236,5 @@ suite("test_remote_doris_unique_table_select", "p0,external") {
235236

236237
sql """ DROP DATABASE IF EXISTS `${db_name}` """
237238
sql """ DROP CATALOG IF EXISTS `${catalog_name}` """
239+
}
238240
}

0 commit comments

Comments
 (0)