Skip to content

Commit 84572d7

Browse files
authored
[feature][insert]add hive table sink thrift (#32274)
add hive table sink thrift add partition type add new compress type issue: #31442
1 parent f359124 commit 84572d7

4 files changed

Lines changed: 34 additions & 15 deletions

File tree

‎gensrc/thrift/DataSinks.thrift‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ enum TDataSinkType {
3838
MULTI_CAST_DATA_STREAM_SINK,
3939
GROUP_COMMIT_OLAP_TABLE_SINK, // deprecated
4040
GROUP_COMMIT_BLOCK_SINK,
41+
HIVE_TABLE_SINK,
4142
}
4243

4344
enum TResultSinkType {
@@ -101,7 +102,7 @@ enum TParquetRepetitionType {
101102
struct TParquetSchema {
102103
1: optional TParquetRepetitionType schema_repetition_type
103104
2: optional TParquetDataType schema_data_type
104-
3: optional string schema_column_name
105+
3: optional string schema_column_name
105106
4: optional TParquetDataLogicalType schema_data_logical_type
106107
}
107108

@@ -282,6 +283,7 @@ struct TOlapTableSink {
282283
struct THiveLocationParams {
283284
1: optional string write_path
284285
2: optional string target_path
286+
3: optional Types.TFileType file_type
285287
}
286288

287289
struct TSortedColumn {
@@ -300,11 +302,16 @@ struct THiveBucket {
300302
4: optional list<TSortedColumn> sorted_by
301303
}
302304

303-
enum THiveCompressionType {
304-
SNAPPY = 3,
305-
LZ4 = 4,
306-
ZLIB = 6,
307-
ZSTD = 7,
305+
enum THiveColumnType {
306+
PARTITION_KEY = 0,
307+
REGULAR = 1,
308+
SYNTHESIZED = 2
309+
}
310+
311+
struct THiveColumn {
312+
1: optional string name
313+
2: optional Types.TTypeDesc data_type
314+
3: optional THiveColumnType column_type
308315
}
309316

310317
struct THivePartition {
@@ -316,13 +323,14 @@ struct THivePartition {
316323
struct THiveTableSink {
317324
1: optional string db_name
318325
2: optional string table_name
319-
3: optional list<string> data_column_names
320-
4: optional list<string> partition_column_names
321-
5: optional list<THivePartition> partitions
322-
6: optional list<THiveBucket> buckets
323-
7: optional PlanNodes.TFileFormatType file_format
324-
8: optional THiveCompressionType compression_type
325-
9: optional THiveLocationParams location
326+
3: optional list<THiveColumn> columns
327+
4: optional list<THivePartition> partitions
328+
5: optional THiveBucket bucket_info
329+
6: optional PlanNodes.TFileFormatType file_format
330+
7: optional PlanNodes.TFileCompressType compression_type
331+
8: optional THiveLocationParams location
332+
9: optional map<string, string> hadoop_config
333+
10: optional bool overwrite
326334
}
327335

328336
enum TUpdateMode {

‎gensrc/thrift/FrontendService.thrift‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ include "Exprs.thrift"
2929
include "RuntimeProfile.thrift"
3030
include "MasterService.thrift"
3131
include "AgentService.thrift"
32+
include "DataSinks.thrift"
3233

3334
// These are supporting structs for JniFrontend.java, which serves as the glue
3435
// between our C++ execution environment and the Java frontend.
@@ -481,6 +482,8 @@ struct TReportExecStatusParams {
481482
24: optional TQueryStatistics query_statistics // deprecated
482483

483484
25: optional TReportWorkloadRuntimeStatusParams report_workload_runtime_status
485+
486+
26: optional list<DataSinks.THivePartitionUpdate> hive_partition_updates
484487
}
485488

486489
struct TFeResult {

‎gensrc/thrift/Partitions.thrift‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,13 @@ enum TPartitionType {
4343
BUCKET_SHFFULE_HASH_PARTITIONED,
4444

4545
// used for shuffle data by parititon and tablet
46-
TABLET_SINK_SHUFFLE_PARTITIONED
46+
TABLET_SINK_SHUFFLE_PARTITIONED,
47+
48+
// used for shuffle data by hive parititon
49+
TABLE_SINK_HASH_PARTITIONED,
50+
51+
// used for hive unparititoned table
52+
TABLE_SINK_RANDOM_PARTITIONED
4753
}
4854

4955
enum TDistributionType {

‎gensrc/thrift/PlanNodes.thrift‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,9 @@ enum TFileCompressType {
138138
DEFLATE,
139139
LZOP,
140140
LZ4BLOCK,
141-
SNAPPYBLOCK
141+
SNAPPYBLOCK,
142+
ZLIB,
143+
ZSTD
142144
}
143145

144146
struct THdfsConf {

0 commit comments

Comments
 (0)