Skip to content

Commit 0deac15

Browse files
authored
[feature](insert)implement hive table sink plan (#31765)
Issue Number: #31442
1 parent 79a8620 commit 0deac15

42 files changed

Lines changed: 1268 additions & 504 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎fe/fe-common/src/main/java/org/apache/doris/common/Config.java‎

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2216,21 +2216,10 @@ public class Config extends ConfigBase {
22162216
"Enable external table DDL"})
22172217
public static boolean enable_external_ddl = false;
22182218

2219-
2220-
@ConfField(mutable = true, masterOnly = true, description = {
2221-
"Hive创建外部表默认指定的input format",
2222-
"Default hive input format for creating table."})
2223-
public static String hive_default_input_format = "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat";
2224-
2225-
@ConfField(mutable = true, masterOnly = true, description = {
2226-
"Hive创建外部表默认指定的output format",
2227-
"Default hive output format for creating table."})
2228-
public static String hive_default_output_format = "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat";
2229-
22302219
@ConfField(mutable = true, masterOnly = true, description = {
2231-
"Hive创建外部表默认指定的SerDe类",
2232-
"Default hive serde class for creating table."})
2233-
public static String hive_default_serde = "org.apache.hadoop.hive.ql.io.orc.OrcSerde";
2220+
"Hive创建外部表默认指定的文件格式",
2221+
"Default hive file format for creating table."})
2222+
public static String hive_default_file_format = "orc";
22342223

22352224
@ConfField
22362225
public static int statistics_sql_parallel_exec_instance_num = 1;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.doris.analysis.CreateTableStmt;
2222
import org.apache.doris.analysis.DropDbStmt;
2323
import org.apache.doris.analysis.DropTableStmt;
24+
import org.apache.doris.analysis.TableName;
2425
import org.apache.doris.catalog.DatabaseIf;
2526
import org.apache.doris.catalog.Env;
2627
import org.apache.doris.catalog.TableIf;
@@ -176,6 +177,8 @@ default CatalogLog constructEditLog() {
176177
return log;
177178
}
178179

180+
TableName getTableNameByTableId(Long tableId);
181+
179182
// Return a copy of all db collection.
180183
Collection<DatabaseIf<? extends TableIf>> getAllDbs();
181184

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.doris.analysis.CreateTableStmt;
2222
import org.apache.doris.analysis.DropDbStmt;
2323
import org.apache.doris.analysis.DropTableStmt;
24+
import org.apache.doris.analysis.TableName;
2425
import org.apache.doris.catalog.Column;
2526
import org.apache.doris.catalog.DatabaseIf;
2627
import org.apache.doris.catalog.Env;
@@ -396,6 +397,16 @@ public List<String> getDbNamesOrEmpty() {
396397
}
397398
}
398399

400+
public TableName getTableNameByTableId(Long tableId) {
401+
for (DatabaseIf<?> db : idToDb.values()) {
402+
TableIf table = db.getTableNullable(tableId);
403+
if (table != null) {
404+
return new TableName(getName(), db.getFullName(), table.getName());
405+
}
406+
}
407+
return null;
408+
}
409+
399410
@Override
400411
public String getResource() {
401412
return catalogProperty.getResource();

‎fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -310,7 +310,7 @@ public TableName getTableNameByTableId(Long tableId) {
310310
for (Database db : fullNameToDb.values()) {
311311
Table table = db.getTableNullable(tableId);
312312
if (table != null) {
313-
return new TableName("", db.getFullName(), table.getName());
313+
return new TableName(INTERNAL_CATALOG_NAME, db.getFullName(), table.getName());
314314
}
315315
}
316316
return null;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCachedClient.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ public interface HMSCachedClient {
5151

5252
List<String> listPartitionNames(String dbName, String tblName);
5353

54+
List<Partition> listPartitions(String dbName, String tblName);
55+
5456
List<String> listPartitionNames(String dbName, String tblName, long maxListPartitionNum);
5557

5658
Partition getPartition(String dbName, String tblName, List<String> partitionValues);

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -125,17 +125,13 @@ public void createTable(CreateTableStmt stmt) throws UserException {
125125
}
126126
try {
127127
Map<String, String> props = stmt.getExtProperties();
128-
String inputFormat = props.getOrDefault("input_format", Config.hive_default_input_format);
129-
String outputFormat = props.getOrDefault("output_format", Config.hive_default_output_format);
130-
String serDe = props.getOrDefault("serde", Config.hive_default_serde);
128+
String fileFormat = props.getOrDefault("file_format", Config.hive_default_file_format);
131129
HiveTableMetadata catalogTable = HiveTableMetadata.of(dbName,
132130
tblName,
133131
stmt.getColumns(),
134132
parsePartitionKeys(props),
135133
props,
136-
inputFormat,
137-
outputFormat,
138-
serDe);
134+
fileFormat);
139135

140136
client.createTable(catalogTable, stmt.isSetIfNotExists());
141137
db.setUnInitialized(true);

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveTableMetadata.java‎

Lines changed: 7 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,7 @@ public class HiveTableMetadata implements TableMetadata {
3030
private String tableName;
3131
private List<Column> columns;
3232
private List<FieldSchema> partitionKeys;
33-
private String inputFormat;
34-
private String outputFormat;
35-
private String serDe;
33+
private String fileFormat;
3634
private Map<String, String> properties;
3735
// private String viewSql;
3836

@@ -41,16 +39,12 @@ public HiveTableMetadata(String dbName,
4139
List<Column> columns,
4240
List<FieldSchema> partitionKeys,
4341
Map<String, String> props,
44-
String inputFormat,
45-
String outputFormat,
46-
String serDe) {
42+
String fileFormat) {
4743
this.dbName = dbName;
4844
this.tableName = tblName;
4945
this.columns = columns;
5046
this.partitionKeys = partitionKeys;
51-
this.inputFormat = inputFormat;
52-
this.outputFormat = outputFormat;
53-
this.serDe = serDe;
47+
this.fileFormat = fileFormat;
5448
this.properties = props;
5549
}
5650

@@ -77,26 +71,16 @@ public List<FieldSchema> getPartitionKeys() {
7771
return partitionKeys;
7872
}
7973

80-
public String getInputFormat() {
81-
return inputFormat;
82-
}
83-
84-
public String getOutputFormat() {
85-
return outputFormat;
86-
}
87-
88-
public String getSerDe() {
89-
return serDe;
74+
public String getFileFormat() {
75+
return fileFormat;
9076
}
9177

9278
public static HiveTableMetadata of(String dbName,
9379
String tblName,
9480
List<Column> columns,
9581
List<FieldSchema> partitionKeys,
9682
Map<String, String> props,
97-
String inputFormat,
98-
String outputFormat, String serDe) {
99-
return new HiveTableMetadata(dbName, tblName, columns, partitionKeys, props,
100-
inputFormat, outputFormat, serDe);
83+
String fileFormat) {
84+
return new HiveTableMetadata(dbName, tblName, columns, partitionKeys, props, fileFormat);
10185
}
10286
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java‎

Lines changed: 24 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,10 @@ public List<String> listPartitionNames(String dbName, String tblName) {
123123
return listPartitionNames(dbName, tblName, (long) -1);
124124
}
125125

126+
public List<Partition> listPartitions(String dbName, String tblName) {
127+
return getPartitionsByNames(dbName, tblName, ImmutableList.of());
128+
}
129+
126130
@Override
127131
public List<String> listPartitionNames(String dbName, String tblName, long maxListPartitionNum) {
128132
String sql = String.format("SELECT \"PART_NAME\" from \"PARTITIONS\" WHERE \"TBL_ID\" = ("
@@ -173,15 +177,26 @@ public List<Partition> getPartitions(String dbName, String tblName, List<String>
173177
private List<Partition> getPartitionsByNames(String dbName, String tblName, List<String> partitionNames) {
174178
List<String> partitionNamesWithQuote = partitionNames.stream().map(partitionName -> "'" + partitionName + "'")
175179
.collect(Collectors.toList());
176-
String partitionNamesString = Joiner.on(", ").join(partitionNamesWithQuote);
177-
String sql = String.format("SELECT \"PART_ID\", \"PARTITIONS\".\"CREATE_TIME\","
178-
+ " \"PARTITIONS\".\"LAST_ACCESS_TIME\","
179-
+ " \"PART_NAME\", \"PARTITIONS\".\"SD_ID\" FROM \"PARTITIONS\""
180-
+ " join \"TBLS\" on \"TBLS\".\"TBL_ID\" = \"PARTITIONS\".\"TBL_ID\""
181-
+ " join \"DBS\" on \"TBLS\".\"DB_ID\" = \"DBS\".\"DB_ID\""
182-
+ " WHERE \"DBS\".\"NAME\" = '%s' AND \"TBLS\".\"TBL_NAME\"='%s'"
183-
+ " AND \"PART_NAME\" in (%s);",
184-
dbName, tblName, partitionNamesString);
180+
String sql;
181+
if (partitionNamesWithQuote.isEmpty()) {
182+
sql = String.format("SELECT \"PART_ID\", \"PARTITIONS\".\"CREATE_TIME\","
183+
+ " \"PARTITIONS\".\"LAST_ACCESS_TIME\","
184+
+ " \"PART_NAME\", \"PARTITIONS\".\"SD_ID\" FROM \"PARTITIONS\""
185+
+ " join \"TBLS\" on \"TBLS\".\"TBL_ID\" = \"PARTITIONS\".\"TBL_ID\""
186+
+ " join \"DBS\" on \"TBLS\".\"DB_ID\" = \"DBS\".\"DB_ID\""
187+
+ " WHERE \"DBS\".\"NAME\" = '%s' AND \"TBLS\".\"TBL_NAME\"='%s';",
188+
dbName, tblName);
189+
} else {
190+
String partitionNamesString = Joiner.on(", ").join(partitionNamesWithQuote);
191+
sql = String.format("SELECT \"PART_ID\", \"PARTITIONS\".\"CREATE_TIME\","
192+
+ " \"PARTITIONS\".\"LAST_ACCESS_TIME\","
193+
+ " \"PART_NAME\", \"PARTITIONS\".\"SD_ID\" FROM \"PARTITIONS\""
194+
+ " join \"TBLS\" on \"TBLS\".\"TBL_ID\" = \"PARTITIONS\".\"TBL_ID\""
195+
+ " join \"DBS\" on \"TBLS\".\"DB_ID\" = \"DBS\".\"DB_ID\""
196+
+ " WHERE \"DBS\".\"NAME\" = '%s' AND \"TBLS\".\"TBL_NAME\"='%s'"
197+
+ " AND \"PART_NAME\" in (%s);",
198+
dbName, tblName, partitionNamesString);
199+
}
185200
if (LOG.isDebugEnabled()) {
186201
LOG.debug("getPartitionsByNames exec sql: {}", sql);
187202
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java‎

Lines changed: 38 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -201,9 +201,7 @@ private static Table toHiveTable(HiveTableMetadata hiveTable) {
201201
// table.setRetention(0);
202202
String location = hiveTable.getProperties().get("external_location");
203203
table.setSd(toHiveStorageDesc(hiveTable.getColumns(),
204-
hiveTable.getInputFormat(),
205-
hiveTable.getOutputFormat(),
206-
hiveTable.getSerDe(),
204+
hiveTable.getFileFormat(),
207205
location));
208206
table.setPartitionKeys(hiveTable.getPartitionKeys());
209207
// table.setViewOriginalText(hiveTable.getViewSql());
@@ -213,15 +211,10 @@ private static Table toHiveTable(HiveTableMetadata hiveTable) {
213211
return table;
214212
}
215213

216-
private static StorageDescriptor toHiveStorageDesc(List<Column> columns, String inputFormat, String outputFormat,
217-
String serDe, String location) {
214+
private static StorageDescriptor toHiveStorageDesc(List<Column> columns, String fileFormat, String location) {
218215
StorageDescriptor sd = new StorageDescriptor();
219216
sd.setCols(toHiveColumns(columns));
220-
SerDeInfo serDeInfo = new SerDeInfo();
221-
serDeInfo.setSerializationLib(serDe);
222-
sd.setSerdeInfo(serDeInfo);
223-
sd.setInputFormat(inputFormat);
224-
sd.setOutputFormat(outputFormat);
217+
setFileFormat(fileFormat, sd);
225218
if (StringUtils.isNotEmpty(location)) {
226219
sd.setLocation(location);
227220
}
@@ -231,6 +224,28 @@ private static StorageDescriptor toHiveStorageDesc(List<Column> columns, String
231224
return sd;
232225
}
233226

227+
private static void setFileFormat(String fileFormat, StorageDescriptor sd) {
228+
String inputFormat;
229+
String outputFormat;
230+
String serDe;
231+
if (fileFormat.equalsIgnoreCase("orc")) {
232+
inputFormat = "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat";
233+
outputFormat = "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat";
234+
serDe = "org.apache.hadoop.hive.ql.io.orc.OrcSerde";
235+
} else if (fileFormat.equalsIgnoreCase("parquet")) {
236+
inputFormat = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat";
237+
outputFormat = "'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat";
238+
serDe = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe";
239+
} else {
240+
throw new IllegalArgumentException("Creating table with an unsupported file format: " + fileFormat);
241+
}
242+
SerDeInfo serDeInfo = new SerDeInfo();
243+
serDeInfo.setSerializationLib(serDe);
244+
sd.setSerdeInfo(serDeInfo);
245+
sd.setInputFormat(inputFormat);
246+
sd.setOutputFormat(outputFormat);
247+
}
248+
234249
private static List<FieldSchema> toHiveColumns(List<Column> columns) {
235250
List<FieldSchema> result = new ArrayList<>();
236251
for (Column column : columns) {
@@ -297,6 +312,19 @@ public List<String> listPartitionNames(String dbName, String tblName) {
297312
return listPartitionNames(dbName, tblName, MAX_LIST_PARTITION_NUM);
298313
}
299314

315+
public List<Partition> listPartitions(String dbName, String tblName) {
316+
try (ThriftHMSClient client = getClient()) {
317+
try {
318+
return ugiDoAs(() -> client.client.listPartitions(dbName, tblName, MAX_LIST_PARTITION_NUM));
319+
} catch (Exception e) {
320+
client.setThrowable(e);
321+
throw e;
322+
}
323+
} catch (Exception e) {
324+
throw new HMSClientException("failed to check if table %s in db %s exists", e, tblName, dbName);
325+
}
326+
}
327+
300328
@Override
301329
public List<String> listPartitionNames(String dbName, String tblName, long maxListPartitionNum) {
302330
// list all parts when the limit is greater than the short maximum

0 commit comments

Comments
 (0)