Skip to content

Commit 460de2d

Browse files
wsjzmorningman
authored andcommitted
[feature][insert]add hive table sink thrift (apache#32274)
add hive table sink thrift add partition type add new compress type issue: apache#31442
1 parent 1645f2e commit 460de2d

4 files changed

Lines changed: 34 additions & 15 deletions

File tree

‎gensrc/thrift/DataSinks.thrift‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ enum TDataSinkType {
3838
MULTI_CAST_DATA_STREAM_SINK,
3939
GROUP_COMMIT_OLAP_TABLE_SINK, // deprecated
4040
GROUP_COMMIT_BLOCK_SINK,
41+
HIVE_TABLE_SINK,
4142
}
4243

4344
enum TResultSinkType {
@@ -101,7 +102,7 @@ enum TParquetRepetitionType {
101102
struct TParquetSchema {
102103
1: optional TParquetRepetitionType schema_repetition_type
103104
2: optional TParquetDataType schema_data_type
104-
3: optional string schema_column_name
105+
3: optional string schema_column_name
105106
4: optional TParquetDataLogicalType schema_data_logical_type
106107
}
107108

@@ -280,6 +281,7 @@ struct TOlapTableSink {
280281
struct THiveLocationParams {
281282
1: optional string write_path
282283
2: optional string target_path
284+
3: optional Types.TFileType file_type
283285
}
284286

285287
struct TSortedColumn {
@@ -298,11 +300,16 @@ struct THiveBucket {
298300
4: optional list<TSortedColumn> sorted_by
299301
}
300302

301-
enum THiveCompressionType {
302-
SNAPPY = 3,
303-
LZ4 = 4,
304-
ZLIB = 6,
305-
ZSTD = 7,
303+
enum THiveColumnType {
304+
PARTITION_KEY = 0,
305+
REGULAR = 1,
306+
SYNTHESIZED = 2
307+
}
308+
309+
struct THiveColumn {
310+
1: optional string name
311+
2: optional Types.TTypeDesc data_type
312+
3: optional THiveColumnType column_type
306313
}
307314

308315
struct THivePartition {
@@ -314,13 +321,14 @@ struct THivePartition {
314321
struct THiveTableSink {
315322
1: optional string db_name
316323
2: optional string table_name
317-
3: optional list<string> data_column_names
318-
4: optional list<string> partition_column_names
319-
5: optional list<THivePartition> partitions
320-
6: optional list<THiveBucket> buckets
321-
7: optional PlanNodes.TFileFormatType file_format
322-
8: optional THiveCompressionType compression_type
323-
9: optional THiveLocationParams location
324+
3: optional list<THiveColumn> columns
325+
4: optional list<THivePartition> partitions
326+
5: optional THiveBucket bucket_info
327+
6: optional PlanNodes.TFileFormatType file_format
328+
7: optional PlanNodes.TFileCompressType compression_type
329+
8: optional THiveLocationParams location
330+
9: optional map<string, string> hadoop_config
331+
10: optional bool overwrite
324332
}
325333

326334
enum TUpdateMode {

‎gensrc/thrift/FrontendService.thrift‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ include "Exprs.thrift"
2929
include "RuntimeProfile.thrift"
3030
include "MasterService.thrift"
3131
include "AgentService.thrift"
32+
include "DataSinks.thrift"
3233

3334
// These are supporting structs for JniFrontend.java, which serves as the glue
3435
// between our C++ execution environment and the Java frontend.
@@ -481,6 +482,8 @@ struct TReportExecStatusParams {
481482
24: optional TQueryStatistics query_statistics // deprecated
482483

483484
25: optional TReportWorkloadRuntimeStatusParams report_workload_runtime_status
485+
486+
26: optional list<DataSinks.THivePartitionUpdate> hive_partition_updates
484487
}
485488

486489
struct TFeResult {

‎gensrc/thrift/Partitions.thrift‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,13 @@ enum TPartitionType {
4343
BUCKET_SHFFULE_HASH_PARTITIONED,
4444

4545
// used for shuffle data by parititon and tablet
46-
TABLET_SINK_SHUFFLE_PARTITIONED
46+
TABLET_SINK_SHUFFLE_PARTITIONED,
47+
48+
// used for shuffle data by hive parititon
49+
TABLE_SINK_HASH_PARTITIONED,
50+
51+
// used for hive unparititoned table
52+
TABLE_SINK_RANDOM_PARTITIONED
4753
}
4854

4955
enum TDistributionType {

‎gensrc/thrift/PlanNodes.thrift‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,9 @@ enum TFileCompressType {
138138
DEFLATE,
139139
LZOP,
140140
LZ4BLOCK,
141-
SNAPPYBLOCK
141+
SNAPPYBLOCK,
142+
ZLIB,
143+
ZSTD
142144
}
143145

144146
struct THdfsConf {

0 commit comments

Comments
 (0)