Skip to content

Commit 27f337b

Browse files
authored
[fix](insert)fix conversion of doris type to hive type (#32735)
#31442 create table fix doris to hive type, use primitiveType to check doris type.
1 parent fbd8346 commit 27f337b

5 files changed

Lines changed: 310 additions & 128 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java‎

Lines changed: 82 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -586,42 +586,89 @@ private static int findNextNestedField(String commaSplitFields) {
586586
* Convert doris type to hive type.
587587
*/
588588
public static String dorisTypeToHiveType(Type dorisType) {
589-
if (dorisType.equals(Type.BOOLEAN)) {
590-
return "boolean";
591-
} else if (dorisType.equals(Type.TINYINT)) {
592-
return "tinyint";
593-
} else if (dorisType.equals(Type.SMALLINT)) {
594-
return "smallint";
595-
} else if (dorisType.equals(Type.INT)) {
596-
return "int";
597-
} else if (dorisType.equals(Type.BIGINT)) {
598-
return "bigint";
599-
} else if (dorisType.equals(Type.DATE) || dorisType.equals(Type.DATEV2)) {
600-
return "date";
601-
} else if (dorisType.equals(Type.DATETIME) || dorisType.equals(Type.DATETIMEV2)) {
602-
return "timestamp";
603-
} else if (dorisType.equals(Type.FLOAT)) {
604-
return "float";
605-
} else if (dorisType.equals(Type.DOUBLE)) {
606-
return "double";
607-
} else if (dorisType.equals(Type.STRING)) {
608-
return "string";
609-
} else if (dorisType.equals(Type.DEFAULT_DECIMALV3)) {
610-
StringBuilder decimalType = new StringBuilder();
611-
decimalType.append("decimal");
612-
ScalarType scalarType = (ScalarType) dorisType;
613-
int precision = scalarType.getScalarPrecision();
614-
if (precision == 0) {
615-
precision = ScalarType.DEFAULT_PRECISION;
589+
if (dorisType.isScalarType()) {
590+
PrimitiveType primitiveType = dorisType.getPrimitiveType();
591+
switch (primitiveType) {
592+
case BOOLEAN:
593+
return "boolean";
594+
case TINYINT:
595+
return "tinyint";
596+
case SMALLINT:
597+
return "smallint";
598+
case INT:
599+
return "int";
600+
case BIGINT:
601+
return "bigint";
602+
case DATEV2:
603+
case DATE:
604+
return "date";
605+
case DATETIMEV2:
606+
case DATETIME:
607+
return "timestamp";
608+
case FLOAT:
609+
return "float";
610+
case DOUBLE:
611+
return "double";
612+
case CHAR: {
613+
ScalarType scalarType = (ScalarType) dorisType;
614+
return "char(" + scalarType.getLength() + ")";
615+
}
616+
case VARCHAR:
617+
case STRING:
618+
return "string";
619+
case DECIMAL32:
620+
case DECIMAL64:
621+
case DECIMAL128:
622+
case DECIMAL256:
623+
case DECIMALV2: {
624+
StringBuilder decimalType = new StringBuilder();
625+
decimalType.append("decimal");
626+
ScalarType scalarType = (ScalarType) dorisType;
627+
int precision = scalarType.getScalarPrecision();
628+
if (precision == 0) {
629+
precision = ScalarType.DEFAULT_PRECISION;
630+
}
631+
// decimal(precision, scale)
632+
int scale = scalarType.getScalarScale();
633+
decimalType.append("(");
634+
decimalType.append(precision);
635+
decimalType.append(",");
636+
decimalType.append(scale);
637+
decimalType.append(")");
638+
return decimalType.toString();
639+
}
640+
default:
641+
throw new HMSClientException("Unsupported primitive type conversion of " + dorisType.toSql());
642+
}
643+
} else if (dorisType.isArrayType()) {
644+
ArrayType dorisArray = (ArrayType) dorisType;
645+
Type itemType = dorisArray.getItemType();
646+
return "array<" + dorisTypeToHiveType(itemType) + ">";
647+
} else if (dorisType.isMapType()) {
648+
MapType dorisMap = (MapType) dorisType;
649+
Type keyType = dorisMap.getKeyType();
650+
Type valueType = dorisMap.getValueType();
651+
return "map<"
652+
+ dorisTypeToHiveType(keyType)
653+
+ ","
654+
+ dorisTypeToHiveType(valueType)
655+
+ ">";
656+
} else if (dorisType.isStructType()) {
657+
StructType dorisStruct = (StructType) dorisType;
658+
StringBuilder structType = new StringBuilder();
659+
structType.append("struct<");
660+
ArrayList<StructField> fields = dorisStruct.getFields();
661+
for (int i = 0; i < fields.size(); i++) {
662+
StructField field = fields.get(i);
663+
structType.append(field.getName());
664+
structType.append(":");
665+
structType.append(dorisTypeToHiveType(field.getType()));
666+
if (i != fields.size() - 1) {
667+
structType.append(",");
668+
}
616669
}
617-
// decimal(precision, scale)
618-
int scale = scalarType.getScalarScale();
619-
decimalType.append("(");
620-
decimalType.append(precision);
621-
decimalType.append(",");
622-
decimalType.append(scale);
623-
decimalType.append(")");
624-
return decimalType.toString();
670+
structType.append(">");
671+
return structType.toString();
625672
}
626673
throw new HMSClientException("Unsupported type conversion of " + dorisType.toSql());
627674
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveUtil.java‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,22 @@
1717

1818
package org.apache.doris.datasource.hive;
1919

20+
import org.apache.doris.catalog.Column;
21+
import org.apache.doris.common.Pair;
2022
import org.apache.doris.common.UserException;
2123
import org.apache.doris.fs.remote.BrokerFileSystem;
2224
import org.apache.doris.fs.remote.RemoteFileSystem;
2325

2426
import com.google.common.base.Preconditions;
2527
import com.google.common.collect.ImmutableList;
2628
import com.google.common.collect.ImmutableMap;
29+
import org.apache.commons.lang3.StringUtils;
2730
import org.apache.hadoop.hive.common.FileUtils;
31+
import org.apache.hadoop.hive.metastore.api.FieldSchema;
2832
import org.apache.hadoop.hive.metastore.api.Partition;
33+
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
34+
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
35+
import org.apache.hadoop.hive.metastore.api.Table;
2936
import org.apache.hadoop.hive.ql.io.SymlinkTextInputFormat;
3037
import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat;
3138
import org.apache.hadoop.mapred.InputFormat;
@@ -36,8 +43,13 @@
3643
import java.io.UnsupportedEncodingException;
3744
import java.net.URLDecoder;
3845
import java.nio.charset.StandardCharsets;
46+
import java.util.ArrayList;
47+
import java.util.HashMap;
48+
import java.util.HashSet;
3949
import java.util.List;
4050
import java.util.Map;
51+
import java.util.Objects;
52+
import java.util.Set;
4153
import java.util.stream.Collectors;
4254

4355
/**
@@ -157,4 +169,87 @@ public static Map<String, Partition> convertToNamePartitionMap(
157169
}
158170
return resultBuilder.build();
159171
}
172+
173+
public static Table toHiveTable(HiveTableMetadata hiveTable) {
174+
Objects.requireNonNull(hiveTable.getDbName(), "Hive database name should be not null");
175+
Objects.requireNonNull(hiveTable.getTableName(), "Hive table name should be not null");
176+
Table table = new Table();
177+
table.setDbName(hiveTable.getDbName());
178+
table.setTableName(hiveTable.getTableName());
179+
// table.setOwner("");
180+
int createTime = (int) System.currentTimeMillis() * 1000;
181+
table.setCreateTime(createTime);
182+
table.setLastAccessTime(createTime);
183+
// table.setRetention(0);
184+
String location = hiveTable.getProperties().get(HiveMetadataOps.LOCATION_URI_KEY);
185+
Set<String> partitionSet = new HashSet<>(hiveTable.getPartitionKeys());
186+
Pair<List<FieldSchema>, List<FieldSchema>> hiveSchema = toHiveSchema(hiveTable.getColumns(), partitionSet);
187+
188+
table.setSd(toHiveStorageDesc(hiveSchema.first, hiveTable.getBucketCols(), hiveTable.getNumBuckets(),
189+
hiveTable.getFileFormat(), location));
190+
table.setPartitionKeys(hiveSchema.second);
191+
192+
// table.setViewOriginalText(hiveTable.getViewSql());
193+
// table.setViewExpandedText(hiveTable.getViewSql());
194+
table.setTableType("MANAGED_TABLE");
195+
table.setParameters(hiveTable.getProperties());
196+
return table;
197+
}
198+
199+
private static StorageDescriptor toHiveStorageDesc(List<FieldSchema> columns,
200+
List<String> bucketCols, int numBuckets, String fileFormat, String location) {
201+
StorageDescriptor sd = new StorageDescriptor();
202+
sd.setCols(columns);
203+
setFileFormat(fileFormat, sd);
204+
if (StringUtils.isNotEmpty(location)) {
205+
sd.setLocation(location);
206+
}
207+
sd.setBucketCols(bucketCols);
208+
sd.setNumBuckets(numBuckets);
209+
Map<String, String> parameters = new HashMap<>();
210+
parameters.put("tag", "doris external hive talbe");
211+
sd.setParameters(parameters);
212+
return sd;
213+
}
214+
215+
private static void setFileFormat(String fileFormat, StorageDescriptor sd) {
216+
String inputFormat;
217+
String outputFormat;
218+
String serDe;
219+
if (fileFormat.equalsIgnoreCase("orc")) {
220+
inputFormat = "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat";
221+
outputFormat = "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat";
222+
serDe = "org.apache.hadoop.hive.ql.io.orc.OrcSerde";
223+
} else if (fileFormat.equalsIgnoreCase("parquet")) {
224+
inputFormat = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat";
225+
outputFormat = "'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat";
226+
serDe = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe";
227+
} else {
228+
throw new IllegalArgumentException("Creating table with an unsupported file format: " + fileFormat);
229+
}
230+
SerDeInfo serDeInfo = new SerDeInfo();
231+
serDeInfo.setSerializationLib(serDe);
232+
sd.setSerdeInfo(serDeInfo);
233+
sd.setInputFormat(inputFormat);
234+
sd.setOutputFormat(outputFormat);
235+
}
236+
237+
private static Pair<List<FieldSchema>, List<FieldSchema>> toHiveSchema(List<Column> columns,
238+
Set<String> partitionSet) {
239+
List<FieldSchema> hiveCols = new ArrayList<>();
240+
List<FieldSchema> hiveParts = new ArrayList<>();
241+
for (Column column : columns) {
242+
FieldSchema hiveFieldSchema = new FieldSchema();
243+
// TODO: add doc, just support doris type
244+
hiveFieldSchema.setType(HiveMetaStoreClientHelper.dorisTypeToHiveType(column.getType()));
245+
hiveFieldSchema.setName(column.getName());
246+
hiveFieldSchema.setComment(column.getComment());
247+
if (partitionSet.contains(column.getName())) {
248+
hiveParts.add(hiveFieldSchema);
249+
} else {
250+
hiveCols.add(hiveFieldSchema);
251+
}
252+
}
253+
return Pair.of(hiveCols, hiveParts);
254+
}
160255
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java‎

Lines changed: 1 addition & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,7 @@
1818
package org.apache.doris.datasource.hive;
1919

2020
import org.apache.doris.analysis.TableName;
21-
import org.apache.doris.catalog.Column;
2221
import org.apache.doris.common.Config;
23-
import org.apache.doris.common.Pair;
2422
import org.apache.doris.datasource.DatabaseMetadata;
2523
import org.apache.doris.datasource.TableMetadata;
2624
import org.apache.doris.datasource.hive.event.MetastoreNotificationFetchException;
@@ -68,18 +66,14 @@
6866
import org.apache.logging.log4j.Logger;
6967

7068
import java.security.PrivilegedExceptionAction;
71-
import java.util.ArrayList;
7269
import java.util.BitSet;
7370
import java.util.Collections;
7471
import java.util.HashMap;
75-
import java.util.HashSet;
7672
import java.util.LinkedList;
7773
import java.util.List;
7874
import java.util.Map;
79-
import java.util.Objects;
8075
import java.util.Optional;
8176
import java.util.Queue;
82-
import java.util.Set;
8377
import java.util.function.Function;
8478
import java.util.stream.Collectors;
8579

@@ -178,7 +172,7 @@ public void createTable(TableMetadata tbl, boolean ignoreIfExists) {
178172
// String location,
179173
if (tbl instanceof HiveTableMetadata) {
180174
ugiDoAs(() -> {
181-
client.client.createTable(toHiveTable((HiveTableMetadata) tbl));
175+
client.client.createTable(HiveUtil.toHiveTable((HiveTableMetadata) tbl));
182176
return null;
183177
});
184178
}
@@ -191,92 +185,6 @@ public void createTable(TableMetadata tbl, boolean ignoreIfExists) {
191185
}
192186
}
193187

194-
private static Table toHiveTable(HiveTableMetadata hiveTable) {
195-
Objects.requireNonNull(hiveTable.getDbName(), "Hive database name should be not null");
196-
Objects.requireNonNull(hiveTable.getTableName(), "Hive table name should be not null");
197-
Table table = new Table();
198-
table.setDbName(hiveTable.getDbName());
199-
table.setTableName(hiveTable.getTableName());
200-
// table.setOwner("");
201-
int createTime = (int) System.currentTimeMillis() * 1000;
202-
table.setCreateTime(createTime);
203-
table.setLastAccessTime(createTime);
204-
// table.setRetention(0);
205-
String location = hiveTable.getProperties().get(HiveMetadataOps.LOCATION_URI_KEY);
206-
Set<String> partitionSet = new HashSet<>(hiveTable.getPartitionKeys());
207-
Pair<List<FieldSchema>, List<FieldSchema>> hiveSchema = toHiveSchema(hiveTable.getColumns(), partitionSet);
208-
209-
table.setSd(toHiveStorageDesc(hiveSchema.first, hiveTable.getBucketCols(), hiveTable.getNumBuckets(),
210-
hiveTable.getFileFormat(), location));
211-
table.setPartitionKeys(hiveSchema.second);
212-
213-
// table.setViewOriginalText(hiveTable.getViewSql());
214-
// table.setViewExpandedText(hiveTable.getViewSql());
215-
table.setTableType("MANAGED_TABLE");
216-
table.setParameters(hiveTable.getProperties());
217-
return table;
218-
}
219-
220-
private static StorageDescriptor toHiveStorageDesc(List<FieldSchema> columns,
221-
List<String> bucketCols,
222-
int numBuckets,
223-
String fileFormat,
224-
String location) {
225-
StorageDescriptor sd = new StorageDescriptor();
226-
sd.setCols(columns);
227-
setFileFormat(fileFormat, sd);
228-
if (StringUtils.isNotEmpty(location)) {
229-
sd.setLocation(location);
230-
}
231-
sd.setBucketCols(bucketCols);
232-
sd.setNumBuckets(numBuckets);
233-
Map<String, String> parameters = new HashMap<>();
234-
parameters.put("tag", "doris external hive talbe");
235-
sd.setParameters(parameters);
236-
return sd;
237-
}
238-
239-
private static void setFileFormat(String fileFormat, StorageDescriptor sd) {
240-
String inputFormat;
241-
String outputFormat;
242-
String serDe;
243-
if (fileFormat.equalsIgnoreCase("orc")) {
244-
inputFormat = "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat";
245-
outputFormat = "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat";
246-
serDe = "org.apache.hadoop.hive.ql.io.orc.OrcSerde";
247-
} else if (fileFormat.equalsIgnoreCase("parquet")) {
248-
inputFormat = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat";
249-
outputFormat = "'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat";
250-
serDe = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe";
251-
} else {
252-
throw new IllegalArgumentException("Creating table with an unsupported file format: " + fileFormat);
253-
}
254-
SerDeInfo serDeInfo = new SerDeInfo();
255-
serDeInfo.setSerializationLib(serDe);
256-
sd.setSerdeInfo(serDeInfo);
257-
sd.setInputFormat(inputFormat);
258-
sd.setOutputFormat(outputFormat);
259-
}
260-
261-
private static Pair<List<FieldSchema>, List<FieldSchema>> toHiveSchema(List<Column> columns,
262-
Set<String> partitionSet) {
263-
List<FieldSchema> hiveCols = new ArrayList<>();
264-
List<FieldSchema> hiveParts = new ArrayList<>();
265-
for (Column column : columns) {
266-
FieldSchema hiveFieldSchema = new FieldSchema();
267-
// TODO: add doc, just support doris type
268-
hiveFieldSchema.setType(HiveMetaStoreClientHelper.dorisTypeToHiveType(column.getType()));
269-
hiveFieldSchema.setName(column.getName());
270-
hiveFieldSchema.setComment(column.getComment());
271-
if (partitionSet.contains(column.getName())) {
272-
hiveParts.add(hiveFieldSchema);
273-
} else {
274-
hiveCols.add(hiveFieldSchema);
275-
}
276-
}
277-
return Pair.of(hiveCols, hiveParts);
278-
}
279-
280188
@Override
281189
public void dropDatabase(String dbName) {
282190
try (ThriftHMSClient client = getClient()) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHiveTableSink.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,7 @@ public PhysicalProperties getRequirePhysicalProperties() {
153153
columnIdx.add(i);
154154
}
155155
}
156+
// mapping partition id
156157
List<ExprId> exprIds = columnIdx.stream()
157158
.map(idx -> child().getOutput().get(idx).getExprId())
158159
.collect(Collectors.toList());

0 commit comments

Comments
 (0)