|
17 | 17 |
|
18 | 18 | package org.apache.doris.datasource.hive; |
19 | 19 |
|
| 20 | +import org.apache.doris.catalog.Column; |
| 21 | +import org.apache.doris.common.Pair; |
20 | 22 | import org.apache.doris.common.UserException; |
21 | 23 | import org.apache.doris.fs.remote.BrokerFileSystem; |
22 | 24 | import org.apache.doris.fs.remote.RemoteFileSystem; |
23 | 25 |
|
24 | 26 | import com.google.common.base.Preconditions; |
25 | 27 | import com.google.common.collect.ImmutableList; |
26 | 28 | import com.google.common.collect.ImmutableMap; |
| 29 | +import org.apache.commons.lang3.StringUtils; |
27 | 30 | import org.apache.hadoop.hive.common.FileUtils; |
| 31 | +import org.apache.hadoop.hive.metastore.api.FieldSchema; |
28 | 32 | import org.apache.hadoop.hive.metastore.api.Partition; |
| 33 | +import org.apache.hadoop.hive.metastore.api.SerDeInfo; |
| 34 | +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; |
| 35 | +import org.apache.hadoop.hive.metastore.api.Table; |
29 | 36 | import org.apache.hadoop.hive.ql.io.SymlinkTextInputFormat; |
30 | 37 | import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat; |
31 | 38 | import org.apache.hadoop.mapred.InputFormat; |
|
36 | 43 | import java.io.UnsupportedEncodingException; |
37 | 44 | import java.net.URLDecoder; |
38 | 45 | import java.nio.charset.StandardCharsets; |
| 46 | +import java.util.ArrayList; |
| 47 | +import java.util.HashMap; |
| 48 | +import java.util.HashSet; |
39 | 49 | import java.util.List; |
40 | 50 | import java.util.Map; |
| 51 | +import java.util.Objects; |
| 52 | +import java.util.Set; |
41 | 53 | import java.util.stream.Collectors; |
42 | 54 |
|
43 | 55 | /** |
@@ -157,4 +169,87 @@ public static Map<String, Partition> convertToNamePartitionMap( |
157 | 169 | } |
158 | 170 | return resultBuilder.build(); |
159 | 171 | } |
| 172 | + |
| 173 | + public static Table toHiveTable(HiveTableMetadata hiveTable) { |
| 174 | + Objects.requireNonNull(hiveTable.getDbName(), "Hive database name should be not null"); |
| 175 | + Objects.requireNonNull(hiveTable.getTableName(), "Hive table name should be not null"); |
| 176 | + Table table = new Table(); |
| 177 | + table.setDbName(hiveTable.getDbName()); |
| 178 | + table.setTableName(hiveTable.getTableName()); |
| 179 | + // table.setOwner(""); |
| 180 | + int createTime = (int) System.currentTimeMillis() * 1000; |
| 181 | + table.setCreateTime(createTime); |
| 182 | + table.setLastAccessTime(createTime); |
| 183 | + // table.setRetention(0); |
| 184 | + String location = hiveTable.getProperties().get(HiveMetadataOps.LOCATION_URI_KEY); |
| 185 | + Set<String> partitionSet = new HashSet<>(hiveTable.getPartitionKeys()); |
| 186 | + Pair<List<FieldSchema>, List<FieldSchema>> hiveSchema = toHiveSchema(hiveTable.getColumns(), partitionSet); |
| 187 | + |
| 188 | + table.setSd(toHiveStorageDesc(hiveSchema.first, hiveTable.getBucketCols(), hiveTable.getNumBuckets(), |
| 189 | + hiveTable.getFileFormat(), location)); |
| 190 | + table.setPartitionKeys(hiveSchema.second); |
| 191 | + |
| 192 | + // table.setViewOriginalText(hiveTable.getViewSql()); |
| 193 | + // table.setViewExpandedText(hiveTable.getViewSql()); |
| 194 | + table.setTableType("MANAGED_TABLE"); |
| 195 | + table.setParameters(hiveTable.getProperties()); |
| 196 | + return table; |
| 197 | + } |
| 198 | + |
| 199 | + private static StorageDescriptor toHiveStorageDesc(List<FieldSchema> columns, |
| 200 | + List<String> bucketCols, int numBuckets, String fileFormat, String location) { |
| 201 | + StorageDescriptor sd = new StorageDescriptor(); |
| 202 | + sd.setCols(columns); |
| 203 | + setFileFormat(fileFormat, sd); |
| 204 | + if (StringUtils.isNotEmpty(location)) { |
| 205 | + sd.setLocation(location); |
| 206 | + } |
| 207 | + sd.setBucketCols(bucketCols); |
| 208 | + sd.setNumBuckets(numBuckets); |
| 209 | + Map<String, String> parameters = new HashMap<>(); |
| 210 | + parameters.put("tag", "doris external hive talbe"); |
| 211 | + sd.setParameters(parameters); |
| 212 | + return sd; |
| 213 | + } |
| 214 | + |
| 215 | + private static void setFileFormat(String fileFormat, StorageDescriptor sd) { |
| 216 | + String inputFormat; |
| 217 | + String outputFormat; |
| 218 | + String serDe; |
| 219 | + if (fileFormat.equalsIgnoreCase("orc")) { |
| 220 | + inputFormat = "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"; |
| 221 | + outputFormat = "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat"; |
| 222 | + serDe = "org.apache.hadoop.hive.ql.io.orc.OrcSerde"; |
| 223 | + } else if (fileFormat.equalsIgnoreCase("parquet")) { |
| 224 | + inputFormat = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"; |
| 225 | + outputFormat = "'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat"; |
| 226 | + serDe = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"; |
| 227 | + } else { |
| 228 | + throw new IllegalArgumentException("Creating table with an unsupported file format: " + fileFormat); |
| 229 | + } |
| 230 | + SerDeInfo serDeInfo = new SerDeInfo(); |
| 231 | + serDeInfo.setSerializationLib(serDe); |
| 232 | + sd.setSerdeInfo(serDeInfo); |
| 233 | + sd.setInputFormat(inputFormat); |
| 234 | + sd.setOutputFormat(outputFormat); |
| 235 | + } |
| 236 | + |
| 237 | + private static Pair<List<FieldSchema>, List<FieldSchema>> toHiveSchema(List<Column> columns, |
| 238 | + Set<String> partitionSet) { |
| 239 | + List<FieldSchema> hiveCols = new ArrayList<>(); |
| 240 | + List<FieldSchema> hiveParts = new ArrayList<>(); |
| 241 | + for (Column column : columns) { |
| 242 | + FieldSchema hiveFieldSchema = new FieldSchema(); |
| 243 | + // TODO: add doc, just support doris type |
| 244 | + hiveFieldSchema.setType(HiveMetaStoreClientHelper.dorisTypeToHiveType(column.getType())); |
| 245 | + hiveFieldSchema.setName(column.getName()); |
| 246 | + hiveFieldSchema.setComment(column.getComment()); |
| 247 | + if (partitionSet.contains(column.getName())) { |
| 248 | + hiveParts.add(hiveFieldSchema); |
| 249 | + } else { |
| 250 | + hiveCols.add(hiveFieldSchema); |
| 251 | + } |
| 252 | + } |
| 253 | + return Pair.of(hiveCols, hiveParts); |
| 254 | + } |
160 | 255 | } |
0 commit comments