Skip to content

Commit 04516fb

Browse files
liugddxxinyiZzz
authored andcommitted
[bugfix](arrow) Fix time zone issues and accuracy issues (apache#38215)
Issue Number: close apache#38174
1 parent 0b98983 commit 04516fb

8 files changed

Lines changed: 47 additions & 21 deletions

File tree

‎be/src/pipeline/exec/memory_scratch_sink_operator.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ Status MemoryScratchSinkOperatorX::sink(RuntimeState* state, vectorized::Block*
104104
{
105105
SCOPED_TIMER(local_state._get_arrow_schema_timer);
106106
// After expr executed, use recaculated schema as final schema
107-
RETURN_IF_ERROR(get_arrow_schema(block, &block_arrow_schema));
107+
RETURN_IF_ERROR(get_arrow_schema(block, &block_arrow_schema, state->timezone()));
108108
}
109109
{
110110
SCOPED_TIMER(local_state._convert_block_to_arrow_batch_timer);

‎be/src/pipeline/exec/result_sink_operator.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include "result_sink_operator.h"
1919

2020
#include <fmt/format.h>
21+
#include <sys/select.h>
2122

2223
#include <memory>
2324

@@ -80,7 +81,8 @@ Status ResultSinkLocalState::open(RuntimeState* state) {
8081
}
8182
case TResultSinkType::ARROW_FLIGHT_PROTOCAL: {
8283
std::shared_ptr<arrow::Schema> arrow_schema;
83-
RETURN_IF_ERROR(convert_expr_ctxs_arrow_schema(_output_vexpr_ctxs, &arrow_schema));
84+
RETURN_IF_ERROR(convert_expr_ctxs_arrow_schema(_output_vexpr_ctxs, &arrow_schema,
85+
state->timezone()));
8486
if (state->query_options().enable_parallel_result_sink) {
8587
state->exec_env()->result_mgr()->register_arrow_schema(state->query_id(), arrow_schema);
8688
} else {

‎be/src/util/arrow/row_batch.cpp‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ namespace doris {
4646

4747
using strings::Substitute;
4848

49-
Status convert_to_arrow_type(const TypeDescriptor& type, std::shared_ptr<arrow::DataType>* result) {
49+
Status convert_to_arrow_type(const TypeDescriptor& type, std::shared_ptr<arrow::DataType>* result,
50+
const std::string& timezone) {
5051
switch (type.type) {
5152
case TYPE_NULL:
5253
*result = arrow::null();
@@ -96,11 +97,11 @@ Status convert_to_arrow_type(const TypeDescriptor& type, std::shared_ptr<arrow::
9697
break;
9798
case TYPE_DATETIMEV2:
9899
if (type.scale > 3) {
99-
*result = std::make_shared<arrow::TimestampType>(arrow::TimeUnit::MICRO);
100+
*result = std::make_shared<arrow::TimestampType>(arrow::TimeUnit::MICRO, timezone);
100101
} else if (type.scale > 0) {
101-
*result = std::make_shared<arrow::TimestampType>(arrow::TimeUnit::MILLI);
102+
*result = std::make_shared<arrow::TimestampType>(arrow::TimeUnit::MILLI, timezone);
102103
} else {
103-
*result = std::make_shared<arrow::TimestampType>(arrow::TimeUnit::SECOND);
104+
*result = std::make_shared<arrow::TimestampType>(arrow::TimeUnit::SECOND, timezone);
104105
}
105106
break;
106107
case TYPE_DECIMALV2:
@@ -120,16 +121,16 @@ Status convert_to_arrow_type(const TypeDescriptor& type, std::shared_ptr<arrow::
120121
case TYPE_ARRAY: {
121122
DCHECK_EQ(type.children.size(), 1);
122123
std::shared_ptr<arrow::DataType> item_type;
123-
static_cast<void>(convert_to_arrow_type(type.children[0], &item_type));
124+
static_cast<void>(convert_to_arrow_type(type.children[0], &item_type, timezone));
124125
*result = std::make_shared<arrow::ListType>(item_type);
125126
break;
126127
}
127128
case TYPE_MAP: {
128129
DCHECK_EQ(type.children.size(), 2);
129130
std::shared_ptr<arrow::DataType> key_type;
130131
std::shared_ptr<arrow::DataType> val_type;
131-
static_cast<void>(convert_to_arrow_type(type.children[0], &key_type));
132-
static_cast<void>(convert_to_arrow_type(type.children[1], &val_type));
132+
static_cast<void>(convert_to_arrow_type(type.children[0], &key_type, timezone));
133+
static_cast<void>(convert_to_arrow_type(type.children[1], &val_type, timezone));
133134
*result = std::make_shared<arrow::MapType>(key_type, val_type);
134135
break;
135136
}
@@ -138,7 +139,7 @@ Status convert_to_arrow_type(const TypeDescriptor& type, std::shared_ptr<arrow::
138139
std::vector<std::shared_ptr<arrow::Field>> fields;
139140
for (size_t i = 0; i < type.children.size(); i++) {
140141
std::shared_ptr<arrow::DataType> field_type;
141-
static_cast<void>(convert_to_arrow_type(type.children[i], &field_type));
142+
static_cast<void>(convert_to_arrow_type(type.children[i], &field_type, timezone));
142143
fields.push_back(std::make_shared<arrow::Field>(type.field_names[i], field_type,
143144
type.contains_nulls[i]));
144145
}
@@ -156,19 +157,21 @@ Status convert_to_arrow_type(const TypeDescriptor& type, std::shared_ptr<arrow::
156157
return Status::OK();
157158
}
158159

159-
Status convert_to_arrow_field(SlotDescriptor* desc, std::shared_ptr<arrow::Field>* field) {
160+
Status convert_to_arrow_field(SlotDescriptor* desc, std::shared_ptr<arrow::Field>* field,
161+
const std::string& timezone) {
160162
std::shared_ptr<arrow::DataType> type;
161-
RETURN_IF_ERROR(convert_to_arrow_type(desc->type(), &type));
163+
RETURN_IF_ERROR(convert_to_arrow_type(desc->type(), &type, timezone));
162164
*field = arrow::field(desc->col_name(), type, desc->is_nullable());
163165
return Status::OK();
164166
}
165167

166-
Status get_arrow_schema(const vectorized::Block& block, std::shared_ptr<arrow::Schema>* result) {
168+
Status get_arrow_schema(const vectorized::Block& block, std::shared_ptr<arrow::Schema>* result,
169+
const std::string& timezone) {
167170
std::vector<std::shared_ptr<arrow::Field>> fields;
168171
for (const auto& type_and_name : block) {
169172
std::shared_ptr<arrow::DataType> arrow_type;
170173
RETURN_IF_ERROR(convert_to_arrow_type(type_and_name.type->get_type_as_type_descriptor(),
171-
&arrow_type));
174+
&arrow_type, timezone));
172175
fields.push_back(std::make_shared<arrow::Field>(type_and_name.name, arrow_type,
173176
type_and_name.type->is_nullable()));
174177
}
@@ -177,12 +180,13 @@ Status get_arrow_schema(const vectorized::Block& block, std::shared_ptr<arrow::S
177180
}
178181

179182
Status convert_expr_ctxs_arrow_schema(const vectorized::VExprContextSPtrs& output_vexpr_ctxs,
180-
std::shared_ptr<arrow::Schema>* result) {
183+
std::shared_ptr<arrow::Schema>* result,
184+
const std::string& timezone) {
181185
std::vector<std::shared_ptr<arrow::Field>> fields;
182186
for (int i = 0; i < output_vexpr_ctxs.size(); i++) {
183187
std::shared_ptr<arrow::DataType> arrow_type;
184188
auto root_expr = output_vexpr_ctxs.at(i)->root();
185-
RETURN_IF_ERROR(convert_to_arrow_type(root_expr->type(), &arrow_type));
189+
RETURN_IF_ERROR(convert_to_arrow_type(root_expr->type(), &arrow_type, timezone));
186190
auto field_name = root_expr->is_slot_ref() && !root_expr->expr_label().empty()
187191
? root_expr->expr_label()
188192
: fmt::format("{}_{}", root_expr->data_type()->get_name(), i);

‎be/src/util/arrow/row_batch.h‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,15 @@ namespace doris {
4141

4242
class RowDescriptor;
4343

44-
Status convert_to_arrow_type(const TypeDescriptor& type, std::shared_ptr<arrow::DataType>* result);
44+
Status convert_to_arrow_type(const TypeDescriptor& type, std::shared_ptr<arrow::DataType>* result,
45+
const std::string& timezone);
4546

46-
Status get_arrow_schema(const vectorized::Block& block, std::shared_ptr<arrow::Schema>* result);
47+
Status get_arrow_schema(const vectorized::Block& block, std::shared_ptr<arrow::Schema>* result,
48+
const std::string& timezone);
4749

4850
Status convert_expr_ctxs_arrow_schema(const vectorized::VExprContextSPtrs& output_vexpr_ctxs,
49-
std::shared_ptr<arrow::Schema>* result);
51+
std::shared_ptr<arrow::Schema>* result,
52+
const std::string& timezone);
5053

5154
Status serialize_record_batch(const arrow::RecordBatch& record_batch, std::string* result);
5255

‎be/src/vec/runtime/vparquet_transformer.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -266,7 +266,8 @@ Status VParquetTransformer::_parse_schema() {
266266
std::vector<std::shared_ptr<arrow::Field>> fields;
267267
for (size_t i = 0; i < _output_vexpr_ctxs.size(); i++) {
268268
std::shared_ptr<arrow::DataType> type;
269-
RETURN_IF_ERROR(convert_to_arrow_type(_output_vexpr_ctxs[i]->root()->type(), &type));
269+
RETURN_IF_ERROR(convert_to_arrow_type(_output_vexpr_ctxs[i]->root()->type(), &type,
270+
_state->timezone()));
270271
if (_parquet_schemas != nullptr) {
271272
std::shared_ptr<arrow::Field> field =
272273
arrow::field(_parquet_schemas->operator[](i).schema_column_name, type,

‎regression-test/data/arrow_flight_sql_p0/test_select.out‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,7 @@
22
-- !arrow_flight_sql --
33
777 4
44

5+
-- !arrow_flight_sql_datetime --
6+
333 plsql333 2024-07-21 12:00:00.123456 2024-07-21 12:00:00.0
7+
222 plsql222 2024-07-20 12:00:00.123456 2024-07-20 12:00:00.0
8+
111 plsql111 2024-07-19 12:00:00.123456 2024-07-19 12:00:00.0

‎regression-test/framework/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -379,7 +379,7 @@ under the License.
379379
<dependency>
380380
<groupId>org.apache.doris</groupId>
381381
<artifactId>flink-doris-connector-1.16</artifactId>
382-
<version>1.6.1</version>
382+
<version>24.0.0</version>
383383
</dependency>
384384
<!-- flink end -->
385385
<dependency>

‎regression-test/suites/arrow_flight_sql_p0/test_select.groovy‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,4 +28,16 @@ suite("test_select", "arrow_flight_sql") {
2828
sql """INSERT INTO ${tableName} VALUES(111, "plsql333")"""
2929

3030
qt_arrow_flight_sql "select sum(id) as a, count(1) as b from ${tableName}"
31+
32+
tableName = "test_select_datetime"
33+
sql "DROP TABLE IF EXISTS ${tableName}"
34+
sql """
35+
create table ${tableName} (id int, name varchar(20), f_datetime_p datetime(6), f_datetime datetime) DUPLICATE key(`id`) distributed by hash (`id`) buckets 4
36+
properties ("replication_num"="1");
37+
"""
38+
sql """INSERT INTO ${tableName} VALUES(111, "plsql111","2024-07-19 12:00:00.123456","2024-07-19 12:00:00")"""
39+
sql """INSERT INTO ${tableName} VALUES(222, "plsql222","2024-07-20 12:00:00.123456","2024-07-20 12:00:00")"""
40+
sql """INSERT INTO ${tableName} VALUES(333, "plsql333","2024-07-21 12:00:00.123456","2024-07-21 12:00:00")"""
41+
42+
qt_arrow_flight_sql_datetime "select * from ${tableName} order by id desc"
3143
}

0 commit comments

Comments
 (0)