Skip to content

Commit 979cb01

Browse files
authored
[Featrue](default value) Support bitmap_empty default value (apache#40364)
1 parent 83f899b commit 979cb01

10 files changed

Lines changed: 204 additions & 7 deletions

File tree

‎be/src/olap/partial_update_info.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include "olap/tablet_meta.h"
2727
#include "olap/tablet_schema.h"
2828
#include "olap/utils.h"
29+
#include "util/bitmap_value.h"
2930
#include "vec/common/assert_cast.h"
3031
#include "vec/core/block.h"
3132

@@ -180,6 +181,11 @@ void PartialUpdateInfo::_generate_default_values_for_missing_cids(
180181
DateV2Value<DateV2ValueType> dv;
181182
dv.from_unixtime(timestamp_ms / 1000, timezone);
182183
default_value = dv.debug_string();
184+
} else if (UNLIKELY(column.type() == FieldType::OLAP_FIELD_TYPE_OBJECT &&
185+
to_lower(column.default_value()).find(to_lower("BITMAP_EMPTY")) !=
186+
std::string::npos)) {
187+
BitmapValue v = BitmapValue {};
188+
default_value = v.to_string();
183189
} else {
184190
default_value = column.default_value();
185191
}

‎fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ BINARY: 'BINARY';
114114
BINLOG: 'BINLOG';
115115
BITAND: 'BITAND';
116116
BITMAP: 'BITMAP';
117+
BITMAP_EMPTY: 'BITMAP_EMPTY';
117118
BITMAP_UNION: 'BITMAP_UNION';
118119
BITOR: 'BITOR';
119120
BITXOR: 'BITXOR';

‎fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1323,7 +1323,7 @@ columnDef
13231323
((GENERATED ALWAYS)? AS LEFT_PAREN generatedExpr=expression RIGHT_PAREN)?
13241324
((NOT)? nullable=NULL)?
13251325
(AUTO_INCREMENT (LEFT_PAREN autoIncInitValue=number RIGHT_PAREN)?)?
1326-
(DEFAULT (nullValue=NULL | INTEGER_VALUE | DECIMAL_VALUE | PI | E | stringValue=STRING_LITERAL
1326+
(DEFAULT (nullValue=NULL | INTEGER_VALUE | DECIMAL_VALUE | PI | E | BITMAP_EMPTY | stringValue=STRING_LITERAL
13271327
| CURRENT_DATE | defaultTimestamp=CURRENT_TIMESTAMP (LEFT_PAREN defaultValuePrecision=number RIGHT_PAREN)?))?
13281328
(ON UPDATE CURRENT_TIMESTAMP (LEFT_PAREN onUpdateValuePrecision=number RIGHT_PAREN)?)?
13291329
(COMMENT comment=STRING_LITERAL)?
@@ -1787,6 +1787,7 @@ nonReserved
17871787
| BIN
17881788
| BITAND
17891789
| BITMAP
1790+
| BITMAP_EMPTY
17901791
| BITMAP_UNION
17911792
| BITOR
17921793
| BITXOR

‎fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -427,8 +427,10 @@ public void analyze(boolean isOlap) throws AnalysisException {
427427
}
428428

429429
if (type.getPrimitiveType() == PrimitiveType.BITMAP) {
430-
if (defaultValue.isSet && defaultValue != DefaultValue.NULL_DEFAULT_VALUE) {
431-
throw new AnalysisException("Bitmap type column can not set default value");
430+
if (defaultValue.isSet && defaultValue != DefaultValue.NULL_DEFAULT_VALUE
431+
&& !defaultValue.value.equals(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE.value)) {
432+
throw new AnalysisException("Bitmap type column default value only support null or "
433+
+ DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE.value);
432434
}
433435
defaultValue = DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE;
434436
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2793,6 +2793,8 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) {
27932793
defaultValue = Optional.of(DefaultValue.PI_DEFAULT_VALUE);
27942794
} else if (ctx.E() != null) {
27952795
defaultValue = Optional.of(DefaultValue.E_NUM_DEFAULT_VALUE);
2796+
} else if (ctx.BITMAP_EMPTY() != null) {
2797+
defaultValue = Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE);
27962798
}
27972799
}
27982800
if (ctx.UPDATE() != null) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -318,13 +318,15 @@ public void validate(boolean isOlap, Set<String> keysSet, Set<String> clusterKey
318318
}
319319
defaultValue = Optional.of(DefaultValue.HLL_EMPTY_DEFAULT_VALUE);
320320
} else if (type.isBitmapType()) {
321-
if (defaultValue.isPresent() && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE) {
322-
throw new AnalysisException("Bitmap type column can not set default value");
321+
if (defaultValue.isPresent() && isOlap && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE
322+
&& !defaultValue.get().getValue().equals(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE.getValue())) {
323+
throw new AnalysisException("Bitmap type column default value only support "
324+
+ DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE);
323325
}
324326
defaultValue = Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE);
325327
} else if (type.isArrayType() && defaultValue.isPresent() && isOlap
326328
&& defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE && !defaultValue.get()
327-
.getValue().equals(DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.getValue())) {
329+
.getValue().equals(DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.getValue())) {
328330
throw new AnalysisException("Array type column default value only support null or "
329331
+ DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE);
330332
} else if (type.isMapType()) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ public class DefaultValue {
3131
public static String CURRENT_TIMESTAMP = "CURRENT_TIMESTAMP";
3232
public static String NOW = "now";
3333
public static String HLL_EMPTY = "HLL_EMPTY";
34+
public static String BITMAP_EMPTY = "BITMAP_EMPTY";
3435
public static DefaultValue CURRENT_DATE_DEFAULT_VALUE = new DefaultValue(CURRENT_DATE, CURRENT_DATE.toLowerCase());
3536
public static DefaultValue CURRENT_TIMESTAMP_DEFAULT_VALUE = new DefaultValue(CURRENT_TIMESTAMP, NOW);
3637
// default null
@@ -40,7 +41,7 @@ public class DefaultValue {
4041
// default "value", "0" means empty hll
4142
public static DefaultValue HLL_EMPTY_DEFAULT_VALUE = new DefaultValue(ZERO, HLL_EMPTY);
4243
// default "value", "0" means empty bitmap
43-
public static DefaultValue BITMAP_EMPTY_DEFAULT_VALUE = new DefaultValue(ZERO);
44+
public static DefaultValue BITMAP_EMPTY_DEFAULT_VALUE = new DefaultValue(ZERO, BITMAP_EMPTY);
4445
// default "value", "[]" means empty array
4546
public static DefaultValue ARRAY_EMPTY_DEFAULT_VALUE = new DefaultValue("[]");
4647
// Lets use the const value from math pacakge.
Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !insert_into1 --
3+
0
4+
0
5+
0
6+
0
7+
8+
-- !stream_load_csv1 --
9+
0
10+
0
11+
0
12+
0
13+
0
14+
0
15+
16+
-- !select_2 --
17+
0
18+
0
19+
0
20+
0
21+
22+
-- !stream_load_csv2 --
23+
0
24+
0
25+
0
26+
0
27+
0
28+
0
29+
30+
-- !insert_into3 --
31+
0
32+
0
33+
0
34+
0
35+
36+
-- !stream_load_csv3 --
37+
0
38+
0
39+
0
40+
0
41+
0
42+
0
43+
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
5,5
2+
6,6
Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
suite("test_default_bitmap_empty") {
19+
sql "SET enable_nereids_planner=true"
20+
sql "SET enable_fallback_to_original_planner=false"
21+
def tableName = "test_default_bitmap_empty"
22+
23+
sql """ DROP TABLE IF EXISTS ${tableName} """
24+
sql """
25+
CREATE TABLE IF NOT EXISTS ${tableName}
26+
(
27+
k TINYINT,
28+
v1 bitmap NOT NULL DEFAULT bitmap_empty,
29+
v2 INT
30+
)
31+
UNIQUE KEY(K)
32+
DISTRIBUTED BY HASH(k)
33+
PROPERTIES("replication_num" = "1");
34+
"""
35+
36+
// test insert into.
37+
sql " insert into ${tableName} (k, v2) values (1, 1); "
38+
sql " insert into ${tableName} (k, v2) values (2, 2); "
39+
sql " insert into ${tableName} (k, v2) values (3, 3); "
40+
sql " insert into ${tableName} (k, v2) values (4, 4); "
41+
sql "sync"
42+
qt_insert_into1 """ select bitmap_count(v1) from ${tableName}; """
43+
44+
// test csv stream load.
45+
streamLoad {
46+
table "${tableName}"
47+
48+
set 'column_separator', ','
49+
set 'columns', 'k, v1=bitmap_empty(), v2'
50+
51+
file 'test_default_bitmap_empty_streamload.csv'
52+
53+
time 10000 // limit inflight 10s
54+
}
55+
56+
sql "sync"
57+
58+
qt_stream_load_csv1 """ select bitmap_count(v1) from ${tableName}; """
59+
60+
// test partial update
61+
sql """ DROP TABLE IF EXISTS ${tableName} """
62+
sql """
63+
CREATE TABLE IF NOT EXISTS ${tableName}
64+
(
65+
k TINYINT,
66+
v1 bitmap NOT NULL DEFAULT bitmap_empty,
67+
v2 INT
68+
)
69+
UNIQUE KEY(K)
70+
DISTRIBUTED BY HASH(k)
71+
PROPERTIES("replication_num" = "1");
72+
"""
73+
74+
sql "set enable_unique_key_partial_update=true;"
75+
sql "set enable_insert_strict=false;"
76+
77+
sql " insert into ${tableName} (k, v2) values (1, 1); "
78+
sql " insert into ${tableName} (k, v2) values (2, 2); "
79+
sql " insert into ${tableName} (k, v2) values (3, 3); "
80+
sql " insert into ${tableName} (k, v2) values (4, 4); "
81+
sql "sync"
82+
83+
qt_select_2 "select bitmap_count(v1) from ${tableName};"
84+
85+
streamLoad {
86+
table "${tableName}"
87+
88+
set 'partial_columns', 'true'
89+
set 'column_separator', ','
90+
set 'columns', 'k, v2'
91+
92+
file 'test_default_bitmap_empty_streamload.csv'
93+
94+
time 10000 // limit inflight 10s
95+
}
96+
97+
sql "sync"
98+
99+
qt_stream_load_csv2 """ select bitmap_count(v1) from ${tableName}; """
100+
101+
sql """ DROP TABLE IF EXISTS ${tableName} """
102+
sql """
103+
CREATE TABLE IF NOT EXISTS ${tableName}
104+
(
105+
k TINYINT,
106+
v1 bitmap BITMAP_UNION default BITMAP_EMPTY,
107+
v2 INT replace_if_not_null
108+
)
109+
aggregate KEY(K)
110+
DISTRIBUTED BY HASH(k)
111+
PROPERTIES("replication_num" = "1");
112+
"""
113+
114+
// test insert into.
115+
sql " insert into ${tableName} (k, v2) values (1, 1); "
116+
sql " insert into ${tableName} (k, v2) values (2, 2); "
117+
sql " insert into ${tableName} (k, v2) values (3, 3); "
118+
sql " insert into ${tableName} (k, v2) values (4, 4); "
119+
sql "sync"
120+
qt_insert_into3 """ select bitmap_count(v1) from ${tableName}; """
121+
122+
// test csv stream load.
123+
streamLoad {
124+
table "${tableName}"
125+
126+
set 'column_separator', ','
127+
set 'columns', 'k, v1=bitmap_empty(), v2'
128+
129+
file 'test_default_bitmap_empty_streamload.csv'
130+
131+
time 10000 // limit inflight 10s
132+
}
133+
134+
sql "sync"
135+
136+
qt_stream_load_csv3 """ select bitmap_count(v1) from ${tableName}; """
137+
}

0 commit comments

Comments
 (0)