Skip to content

Commit e81ca0e

Browse files
wsjzHappenLee
authored andcommitted
[feature](insert)support default value when create hive table (apache#33666)
Issue Number: apache#31442 hive3 support create table with column's default value if use hive3, we can write default value to table
1 parent 02927fb commit e81ca0e

10 files changed

Lines changed: 280 additions & 125 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,12 @@ public Column(String name, Type type, boolean isKey, AggregateType aggregateType
193193
false, null);
194194
}
195195

196+
public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull,
197+
String defaultValue, String comment, boolean visible, int colUniqueId) {
198+
this(name, type, isKey, aggregateType, isAllowNull, -1, defaultValue, comment, visible, null, colUniqueId, null,
199+
false, null);
200+
}
201+
196202
public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull,
197203
String defaultValue, String comment, boolean visible, DefaultValueExprDef defaultValueExprDef,
198204
int colUniqueId, String realDefaultValue) {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCachedClient.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@ public interface HMSCachedClient {
6363

6464
List<FieldSchema> getSchema(String dbName, String tblName);
6565

66+
Map<String, String> getDefaultColumnValues(String dbName, String tblName);
67+
6668
List<ColumnStatisticsObj> getTableColumnStatistics(String dbName, String tblName,
6769
List<String> columns);
6870

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -584,13 +584,17 @@ private List<Column> getHudiSchema() {
584584
}
585585

586586
private List<Column> getHiveSchema() {
587+
HMSCachedClient client = ((HMSExternalCatalog) catalog).getClient();
587588
List<Column> columns;
588-
List<FieldSchema> schema = ((HMSExternalCatalog) catalog).getClient().getSchema(dbName, name);
589+
List<FieldSchema> schema = client.getSchema(dbName, name);
590+
Map<String, String> colDefaultValues = client.getDefaultColumnValues(dbName, name);
589591
List<Column> tmpSchema = Lists.newArrayListWithCapacity(schema.size());
590592
for (FieldSchema field : schema) {
591-
tmpSchema.add(new Column(field.getName().toLowerCase(Locale.ROOT),
593+
String fieldName = field.getName().toLowerCase(Locale.ROOT);
594+
String defaultValue = colDefaultValues.getOrDefault(fieldName, null);
595+
tmpSchema.add(new Column(fieldName,
592596
HiveMetaStoreClientHelper.hiveTypeToDorisType(field.getType()), true, null,
593-
true, field.getComment(), true, -1));
597+
true, defaultValue, field.getComment(), true, -1));
594598
}
595599
columns = tmpSchema;
596600
return columns;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveVersionUtil.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ public static HiveVersion getVersion(String version) {
7171
return DEFAULT_HIVE_VERSION;
7272
}
7373
} else if (major >= 3) {
74-
return HiveVersion.V2_3;
74+
return HiveVersion.V3_0;
7575
} else {
7676
LOG.warn("invalid hive version: " + version);
7777
return DEFAULT_HIVE_VERSION;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import java.sql.Connection;
5050
import java.sql.PreparedStatement;
5151
import java.sql.ResultSet;
52+
import java.util.HashMap;
5253
import java.util.List;
5354
import java.util.Map;
5455
import java.util.function.Function;
@@ -245,6 +246,15 @@ private List<String> getPartitionValues(int partitionId) {
245246
}
246247
}
247248

249+
@Override
250+
public Map<String, String> getDefaultColumnValues(String dbName, String tblName) {
251+
if (LOG.isDebugEnabled()) {
252+
LOG.debug("Do not support default column values in PostgreSQLJdbcHMSCachedClient."
253+
+ " Will use null values instead.");
254+
}
255+
return new HashMap<>();
256+
}
257+
248258
@Override
249259
public Table getTable(String dbName, String tblName) {
250260
String sql = "SELECT \"TBL_ID\", \"TBL_NAME\", \"DBS\".\"NAME\", \"OWNER\", \"CREATE_TIME\","

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java‎

Lines changed: 59 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.doris.datasource.hive;
1919

2020
import org.apache.doris.analysis.TableName;
21+
import org.apache.doris.catalog.Column;
2122
import org.apache.doris.common.Config;
2223
import org.apache.doris.datasource.DatabaseMetadata;
2324
import org.apache.doris.datasource.TableMetadata;
@@ -46,24 +47,30 @@
4647
import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
4748
import org.apache.hadoop.hive.metastore.api.DataOperationType;
4849
import org.apache.hadoop.hive.metastore.api.Database;
50+
import org.apache.hadoop.hive.metastore.api.DefaultConstraintsRequest;
4951
import org.apache.hadoop.hive.metastore.api.FieldSchema;
5052
import org.apache.hadoop.hive.metastore.api.LockComponent;
5153
import org.apache.hadoop.hive.metastore.api.LockResponse;
5254
import org.apache.hadoop.hive.metastore.api.LockState;
5355
import org.apache.hadoop.hive.metastore.api.MetaException;
5456
import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
5557
import org.apache.hadoop.hive.metastore.api.Partition;
58+
import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint;
5659
import org.apache.hadoop.hive.metastore.api.Table;
5760
import org.apache.hadoop.hive.metastore.api.TableValidWriteIds;
5861
import org.apache.hadoop.hive.metastore.txn.TxnUtils;
5962
import org.apache.logging.log4j.LogManager;
6063
import org.apache.logging.log4j.Logger;
64+
import shade.doris.hive.org.apache.thrift.TApplicationException;
6165

6266
import java.security.PrivilegedExceptionAction;
67+
import java.util.ArrayList;
6368
import java.util.BitSet;
6469
import java.util.Collections;
70+
import java.util.HashMap;
6571
import java.util.LinkedList;
6672
import java.util.List;
73+
import java.util.Locale;
6774
import java.util.Map;
6875
import java.util.Optional;
6976
import java.util.Queue;
@@ -153,8 +160,28 @@ public void createTable(TableMetadata tbl, boolean ignoreIfExists) {
153160
try {
154161
// String location,
155162
if (tbl instanceof HiveTableMetadata) {
163+
Table hiveTable = HiveUtil.toHiveTable((HiveTableMetadata) tbl);
164+
List<Column> tableColumns = ((HiveTableMetadata) tbl).getColumns();
165+
List<SQLDefaultConstraint> dvs = new ArrayList<>(tableColumns.size());
166+
for (Column tableColumn : tableColumns) {
167+
if (tableColumn.hasDefaultValue()) {
168+
SQLDefaultConstraint dv = new SQLDefaultConstraint();
169+
dv.setTable_db(tbl.getDbName());
170+
dv.setTable_name(tbl.getTableName());
171+
dv.setColumn_name(tableColumn.getName());
172+
dv.setDefault_value(tableColumn.getDefaultValue());
173+
dv.setDc_name(tableColumn.getName() + "_dv_constraint");
174+
dvs.add(dv);
175+
}
176+
}
156177
ugiDoAs(() -> {
157-
client.client.createTable(HiveUtil.toHiveTable((HiveTableMetadata) tbl));
178+
if (!dvs.isEmpty()) {
179+
// foreignKeys, uniqueConstraints, notNullConstraints, defaultConstraints, checkConstraints
180+
client.client.createTableWithConstraints(hiveTable, null,
181+
null, null, null, dvs, null);
182+
return null;
183+
}
184+
client.client.createTable(hiveTable);
158185
return null;
159186
});
160187
}
@@ -293,6 +320,37 @@ public Database getDatabase(String dbName) {
293320
}
294321
}
295322

323+
public Map<String, String> getDefaultColumnValues(String dbName, String tblName) {
324+
Map<String, String> res = new HashMap<>();
325+
try (ThriftHMSClient client = getClient()) {
326+
try {
327+
DefaultConstraintsRequest req = new DefaultConstraintsRequest();
328+
req.setDb_name(dbName);
329+
req.setTbl_name(tblName);
330+
List<SQLDefaultConstraint> dvcs = ugiDoAs(() -> {
331+
try {
332+
return client.client.getDefaultConstraints(req);
333+
} catch (TApplicationException e) {
334+
if (e.getMessage().contains("Invalid method name: 'get_default_constraints'")) {
335+
// the getDefaultConstraints method only supported on hive3
336+
return ImmutableList.of();
337+
}
338+
throw e;
339+
}
340+
});
341+
for (SQLDefaultConstraint dvc : dvcs) {
342+
res.put(dvc.getColumn_name().toLowerCase(Locale.ROOT), dvc.getDefault_value());
343+
}
344+
return res;
345+
} catch (Exception e) {
346+
client.setThrowable(e);
347+
throw e;
348+
}
349+
} catch (Exception e) {
350+
throw new HMSClientException("failed to get table %s in db %s from hms client", e, tblName, dbName);
351+
}
352+
}
353+
296354
@Override
297355
public Table getTable(String dbName, String tblName) {
298356
try (ThriftHMSClient client = getClient()) {

‎fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1089,7 +1089,10 @@ public void createTableWithConstraints(Table tbl,
10891089
List<SQLCheckConstraint> checkConstraints)
10901090
throws AlreadyExistsException, InvalidObjectException,
10911091
MetaException, NoSuchObjectException, TException {
1092-
1092+
if (hiveVersion != HiveVersion.V3_0) {
1093+
throw new UnsupportedOperationException("Table with default values is not supported "
1094+
+ "if the hive version is less than 3.0. Can set 'hive.version' to 3.0 in properties.");
1095+
}
10931096
if (!tbl.isSetCatName()) {
10941097
String defaultCat = getDefaultCatalog(conf);
10951098
tbl.setCatName(defaultCat);

‎fe/fe-core/src/test/java/org/apache/doris/datasource/TestHMSCachedClient.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,11 @@ public List<FieldSchema> getSchema(String dbName, String tblName) {
162162
return null;
163163
}
164164

165+
@Override
166+
public Map<String, String> getDefaultColumnValues(String dbName, String tblName) {
167+
return new HashMap<>();
168+
}
169+
165170
@Override
166171
public List<ColumnStatisticsObj> getTableColumnStatistics(String dbName, String tblName, List<String> columns) {
167172
return null;

‎regression-test/suites/external_table_p0/hive/ddl/test_hive_ctas.groovy‎

Lines changed: 34 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -396,22 +396,43 @@ suite("test_hive_ctas", "p0,external,hive,external_docker,external_docker_hive")
396396
sql """ switch `${catalog_name}` """
397397
sql """ CREATE DATABASE IF NOT EXISTS `test_ctas_all_type` """;
398398
sql """ use test_ctas_all_type """;
399+
// TODO: work on hive3
400+
// sql """
401+
// CREATE TABLE IF NOT EXISTS all_types_ctas_${file_format}_with_dv(
402+
// `col1` BOOLEAN DEFAULT 'false' COMMENT 'col1',
403+
// `col2` TINYINT DEFAULT '127' COMMENT 'col2',
404+
// `col3` SMALLINT DEFAULT '32767' COMMENT 'col3',
405+
// `col4` INT DEFAULT '2147483647' COMMENT 'col4',
406+
// `col5` BIGINT DEFAULT '9223372036854775807' COMMENT 'col5',
407+
// `col6` CHAR(10) DEFAULT 'default' COMMENT 'col6',
408+
// `col7` FLOAT DEFAULT '1' COMMENT 'col7',
409+
// `col8` DOUBLE DEFAULT '3.141592653' COMMENT 'col8',
410+
// `col9` DECIMAL(9,4) DEFAULT '99999.9999' COMMENT 'col9',
411+
// `col10` VARCHAR(11) DEFAULT 'default' COMMENT 'col10',
412+
// `col11` STRING DEFAULT 'default' COMMENT 'col11',
413+
// `col12` DATE DEFAULT '2023-05-29' COMMENT 'col12',
414+
// `col13` DATETIME DEFAULT current_timestamp COMMENT 'col13'
415+
// ) ENGINE=hive
416+
// PROPERTIES (
417+
// 'file_format'='${file_format}'
418+
// )
419+
// """
399420

400421
sql """
401422
CREATE TABLE IF NOT EXISTS all_types_ctas_${file_format}(
402-
`col1` BOOLEAN DEFAULT 'false' COMMENT 'col1',
403-
`col2` TINYINT DEFAULT '127' COMMENT 'col2',
404-
`col3` SMALLINT DEFAULT '32767' COMMENT 'col3',
405-
`col4` INT DEFAULT '2147483647' COMMENT 'col4',
406-
`col5` BIGINT DEFAULT '9223372036854775807' COMMENT 'col5',
407-
`col6` CHAR(10) DEFAULT 'default' COMMENT 'col6',
408-
`col7` FLOAT DEFAULT '1' COMMENT 'col7',
409-
`col8` DOUBLE DEFAULT '3.141592653' COMMENT 'col8',
410-
`col9` DECIMAL(9,4) DEFAULT '99999.9999' COMMENT 'col9',
411-
`col10` VARCHAR(11) DEFAULT 'default' COMMENT 'col10',
412-
`col11` STRING DEFAULT 'default' COMMENT 'col11',
413-
`col12` DATE DEFAULT '2023-05-29' COMMENT 'col12',
414-
`col13` DATETIME DEFAULT current_timestamp COMMENT 'col13'
423+
`col1` BOOLEAN COMMENT 'col1',
424+
`col2` TINYINT COMMENT 'col2',
425+
`col3` SMALLINT COMMENT 'col3',
426+
`col4` INT COMMENT 'col4',
427+
`col5` BIGINT COMMENT 'col5',
428+
`col6` CHAR(10) COMMENT 'col6',
429+
`col7` FLOAT COMMENT 'col7',
430+
`col8` DOUBLE COMMENT 'col8',
431+
`col9` DECIMAL(9,4) COMMENT 'col9',
432+
`col10` VARCHAR(11) COMMENT 'col10',
433+
`col11` STRING COMMENT 'col11',
434+
`col12` DATE COMMENT 'col12',
435+
`col13` DATETIME COMMENT 'col13'
415436
) ENGINE=hive
416437
PROPERTIES (
417438
'file_format'='${file_format}'

0 commit comments

Comments
 (0)