Skip to content

Commit 3794034

Browse files
xinyiZzzseawinde
authored andcommitted
[feature-wip](arrow-flight)(step4) Support other DML and DDL statements, besides Select (apache#25919)
Design Documentation Linked to apache#25514
1 parent e507335 commit 3794034

30 files changed

Lines changed: 1137 additions & 600 deletions

‎be/src/runtime/result_buffer_mgr.cpp‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -109,21 +109,21 @@ std::shared_ptr<BufferControlBlock> ResultBufferMgr::find_control_block(const TU
109109
return std::shared_ptr<BufferControlBlock>();
110110
}
111111

112-
void ResultBufferMgr::register_row_descriptor(const TUniqueId& query_id,
113-
const RowDescriptor& row_desc) {
114-
std::unique_lock<std::shared_mutex> wlock(_row_descriptor_map_lock);
115-
_row_descriptor_map.insert(std::make_pair(query_id, row_desc));
112+
void ResultBufferMgr::register_arrow_schema(const TUniqueId& query_id,
113+
const std::shared_ptr<arrow::Schema>& arrow_schema) {
114+
std::unique_lock<std::shared_mutex> wlock(_arrow_schema_map_lock);
115+
_arrow_schema_map.insert(std::make_pair(query_id, arrow_schema));
116116
}
117117

118-
RowDescriptor ResultBufferMgr::find_row_descriptor(const TUniqueId& query_id) {
119-
std::shared_lock<std::shared_mutex> rlock(_row_descriptor_map_lock);
120-
RowDescriptorMap::iterator iter = _row_descriptor_map.find(query_id);
118+
std::shared_ptr<arrow::Schema> ResultBufferMgr::find_arrow_schema(const TUniqueId& query_id) {
119+
std::shared_lock<std::shared_mutex> rlock(_arrow_schema_map_lock);
120+
auto iter = _arrow_schema_map.find(query_id);
121121

122-
if (_row_descriptor_map.end() != iter) {
122+
if (_arrow_schema_map.end() != iter) {
123123
return iter->second;
124124
}
125125

126-
return RowDescriptor();
126+
return nullptr;
127127
}
128128

129129
void ResultBufferMgr::fetch_data(const PUniqueId& finst_id, GetResultBatchCtx* ctx) {
@@ -162,11 +162,11 @@ Status ResultBufferMgr::cancel(const TUniqueId& query_id) {
162162
}
163163

164164
{
165-
std::unique_lock<std::shared_mutex> wlock(_row_descriptor_map_lock);
166-
RowDescriptorMap::iterator row_desc_iter = _row_descriptor_map.find(query_id);
165+
std::unique_lock<std::shared_mutex> wlock(_arrow_schema_map_lock);
166+
auto arrow_schema_iter = _arrow_schema_map.find(query_id);
167167

168-
if (_row_descriptor_map.end() != row_desc_iter) {
169-
_row_descriptor_map.erase(row_desc_iter);
168+
if (_arrow_schema_map.end() != arrow_schema_iter) {
169+
_arrow_schema_map.erase(arrow_schema_iter);
170170
}
171171
}
172172

‎be/src/runtime/result_buffer_mgr.h‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,12 @@
2929

3030
#include "common/status.h"
3131
#include "gutil/ref_counted.h"
32-
#include "runtime/descriptors.h"
3332
#include "util/countdown_latch.h"
3433
#include "util/hash_util.hpp"
3534

3635
namespace arrow {
3736
class RecordBatch;
37+
class Schema;
3838
} // namespace arrow
3939

4040
namespace doris {
@@ -66,8 +66,9 @@ class ResultBufferMgr {
6666
// fetch data result to Arrow Flight Server
6767
Status fetch_arrow_data(const TUniqueId& finst_id, std::shared_ptr<arrow::RecordBatch>* result);
6868

69-
void register_row_descriptor(const TUniqueId& query_id, const RowDescriptor& row_desc);
70-
RowDescriptor find_row_descriptor(const TUniqueId& query_id);
69+
void register_arrow_schema(const TUniqueId& query_id,
70+
const std::shared_ptr<arrow::Schema>& arrow_schema);
71+
std::shared_ptr<arrow::Schema> find_arrow_schema(const TUniqueId& query_id);
7172

7273
// cancel
7374
Status cancel(const TUniqueId& fragment_id);
@@ -78,7 +79,7 @@ class ResultBufferMgr {
7879
private:
7980
using BufferMap = std::unordered_map<TUniqueId, std::shared_ptr<BufferControlBlock>>;
8081
using TimeoutMap = std::map<time_t, std::vector<TUniqueId>>;
81-
using RowDescriptorMap = std::unordered_map<TUniqueId, RowDescriptor>;
82+
using ArrowSchemaMap = std::unordered_map<TUniqueId, std::shared_ptr<arrow::Schema>>;
8283

8384
std::shared_ptr<BufferControlBlock> find_control_block(const TUniqueId& query_id);
8485

@@ -90,10 +91,10 @@ class ResultBufferMgr {
9091
std::shared_mutex _buffer_map_lock;
9192
// buffer block map
9293
BufferMap _buffer_map;
93-
// lock for descriptor map
94-
std::shared_mutex _row_descriptor_map_lock;
94+
// lock for arrow schema map
95+
std::shared_mutex _arrow_schema_map_lock;
9596
// for arrow flight
96-
RowDescriptorMap _row_descriptor_map;
97+
ArrowSchemaMap _arrow_schema_map;
9798

9899
// lock for timeout map
99100
std::mutex _timeout_lock;

‎be/src/service/arrow_flight/arrow_flight_batch_reader.cpp‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -39,18 +39,12 @@ ArrowFlightBatchReader::ArrowFlightBatchReader(std::shared_ptr<QueryStatement> s
3939
arrow::Result<std::shared_ptr<ArrowFlightBatchReader>> ArrowFlightBatchReader::Create(
4040
const std::shared_ptr<QueryStatement>& statement_) {
4141
// Make sure that FE send the fragment to BE and creates the BufferControlBlock before returning ticket
42-
// to the ADBC client, so that the row_descriptor and control block can be found.
43-
RowDescriptor row_desc =
44-
ExecEnv::GetInstance()->result_mgr()->find_row_descriptor(statement_->query_id);
45-
if (row_desc.equals(RowDescriptor())) {
42+
// to the ADBC client, so that the schema and control block can be found.
43+
auto schema = ExecEnv::GetInstance()->result_mgr()->find_arrow_schema(statement_->query_id);
44+
if (schema == nullptr) {
4645
ARROW_RETURN_NOT_OK(arrow::Status::Invalid(fmt::format(
47-
"Schema RowDescriptor Not Found, queryid: {}", print_id(statement_->query_id))));
48-
}
49-
std::shared_ptr<arrow::Schema> schema;
50-
auto st = convert_to_arrow_schema(row_desc, &schema);
51-
if (UNLIKELY(!st.ok())) {
52-
LOG(WARNING) << st.to_string();
53-
ARROW_RETURN_NOT_OK(to_arrow_status(st));
46+
"not found arrow flight schema, maybe query has been canceled, queryid: {}",
47+
print_id(statement_->query_id))));
5448
}
5549
std::shared_ptr<ArrowFlightBatchReader> result(new ArrowFlightBatchReader(statement_, schema));
5650
return result;

‎be/src/service/internal_service.cpp‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -718,23 +718,19 @@ void PInternalServiceImpl::fetch_arrow_flight_schema(google::protobuf::RpcContro
718718
google::protobuf::Closure* done) {
719719
bool ret = _light_work_pool.try_offer([request, result, done]() {
720720
brpc::ClosureGuard closure_guard(done);
721-
RowDescriptor row_desc = ExecEnv::GetInstance()->result_mgr()->find_row_descriptor(
722-
UniqueId(request->finst_id()).to_thrift());
723-
if (row_desc.equals(RowDescriptor())) {
724-
auto st = Status::NotFound("not found row descriptor");
725-
st.to_protobuf(result->mutable_status());
726-
return;
727-
}
728-
729-
std::shared_ptr<arrow::Schema> schema;
730-
auto st = convert_to_arrow_schema(row_desc, &schema);
731-
if (UNLIKELY(!st.ok())) {
721+
std::shared_ptr<arrow::Schema> schema =
722+
ExecEnv::GetInstance()->result_mgr()->find_arrow_schema(
723+
UniqueId(request->finst_id()).to_thrift());
724+
if (schema == nullptr) {
725+
LOG(INFO) << "not found arrow flight schema, maybe query has been canceled";
726+
auto st = Status::NotFound(
727+
"not found arrow flight schema, maybe query has been canceled");
732728
st.to_protobuf(result->mutable_status());
733729
return;
734730
}
735731

736732
std::string schema_str;
737-
st = serialize_arrow_schema(row_desc, &schema, &schema_str);
733+
auto st = serialize_arrow_schema(&schema, &schema_str);
738734
if (st.ok()) {
739735
result->set_schema(std::move(schema_str));
740736
}

‎be/src/util/arrow/row_batch.cpp‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@
3939
#include "runtime/types.h"
4040
#include "util/arrow/block_convertor.h"
4141
#include "vec/core/block.h"
42+
#include "vec/exprs/vexpr.h"
43+
#include "vec/exprs/vexpr_context.h"
4244

4345
namespace doris {
4446

@@ -163,6 +165,22 @@ Status convert_to_arrow_schema(const RowDescriptor& row_desc,
163165
return Status::OK();
164166
}
165167

168+
Status convert_expr_ctxs_arrow_schema(const vectorized::VExprContextSPtrs& output_vexpr_ctxs,
169+
std::shared_ptr<arrow::Schema>* result) {
170+
std::vector<std::shared_ptr<arrow::Field>> fields;
171+
for (auto expr_ctx : output_vexpr_ctxs) {
172+
std::shared_ptr<arrow::DataType> arrow_type;
173+
auto root_expr = expr_ctx->root();
174+
RETURN_IF_ERROR(convert_to_arrow_type(root_expr->type(), &arrow_type));
175+
auto field_name = root_expr->is_slot_ref() ? root_expr->expr_name()
176+
: root_expr->data_type()->get_name();
177+
fields.push_back(
178+
std::make_shared<arrow::Field>(field_name, arrow_type, root_expr->is_nullable()));
179+
}
180+
*result = arrow::schema(std::move(fields));
181+
return Status::OK();
182+
}
183+
166184
Status serialize_record_batch(const arrow::RecordBatch& record_batch, std::string* result) {
167185
// create sink memory buffer outputstream with the computed capacity
168186
int64_t capacity;
@@ -206,15 +224,13 @@ Status serialize_record_batch(const arrow::RecordBatch& record_batch, std::strin
206224
return Status::OK();
207225
}
208226

209-
Status serialize_arrow_schema(RowDescriptor row_desc, std::shared_ptr<arrow::Schema>* schema,
210-
std::string* result) {
211-
std::vector<SlotDescriptor*> slots;
212-
for (auto tuple_desc : row_desc.tuple_descriptors()) {
213-
slots.insert(slots.end(), tuple_desc->slots().begin(), tuple_desc->slots().end());
227+
Status serialize_arrow_schema(std::shared_ptr<arrow::Schema>* schema, std::string* result) {
228+
auto make_empty_result = arrow::RecordBatch::MakeEmpty(*schema);
229+
if (!make_empty_result.ok()) {
230+
return Status::InternalError("serialize_arrow_schema failed, reason: {}",
231+
make_empty_result.status().ToString());
214232
}
215-
auto block = vectorized::Block(slots, 0);
216-
std::shared_ptr<arrow::RecordBatch> batch;
217-
RETURN_IF_ERROR(convert_to_arrow_batch(block, *schema, arrow::default_memory_pool(), &batch));
233+
auto batch = make_empty_result.ValueOrDie();
218234
return serialize_record_batch(*batch, result);
219235
}
220236

‎be/src/util/arrow/row_batch.h‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include "common/status.h"
2424
#include "runtime/types.h"
2525
#include "vec/core/block.h"
26+
#include "vec/exprs/vexpr_fwd.h"
2627

2728
// This file will convert Doris RowBatch to/from Arrow's RecordBatch
2829
// RowBatch is used by Doris query engine to exchange data between
@@ -49,9 +50,11 @@ Status convert_to_arrow_schema(const RowDescriptor& row_desc,
4950
Status convert_block_arrow_schema(const vectorized::Block& block,
5051
std::shared_ptr<arrow::Schema>* result);
5152

53+
Status convert_expr_ctxs_arrow_schema(const vectorized::VExprContextSPtrs& output_vexpr_ctxs,
54+
std::shared_ptr<arrow::Schema>* result);
55+
5256
Status serialize_record_batch(const arrow::RecordBatch& record_batch, std::string* result);
5357

54-
Status serialize_arrow_schema(RowDescriptor row_desc, std::shared_ptr<arrow::Schema>* schema,
55-
std::string* result);
58+
Status serialize_arrow_schema(std::shared_ptr<arrow::Schema>* schema, std::string* result);
5659

5760
} // namespace doris

‎be/src/vec/sink/varrow_flight_result_writer.cpp‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,23 +27,20 @@
2727
namespace doris {
2828
namespace vectorized {
2929

30-
VArrowFlightResultWriter::VArrowFlightResultWriter(BufferControlBlock* sinker,
31-
const VExprContextSPtrs& output_vexpr_ctxs,
32-
RuntimeProfile* parent_profile,
33-
const RowDescriptor& row_desc)
30+
VArrowFlightResultWriter::VArrowFlightResultWriter(
31+
BufferControlBlock* sinker, const VExprContextSPtrs& output_vexpr_ctxs,
32+
RuntimeProfile* parent_profile, const std::shared_ptr<arrow::Schema>& arrow_schema)
3433
: _sinker(sinker),
3534
_output_vexpr_ctxs(output_vexpr_ctxs),
3635
_parent_profile(parent_profile),
37-
_row_desc(row_desc) {}
36+
_arrow_schema(arrow_schema) {}
3837

3938
Status VArrowFlightResultWriter::init(RuntimeState* state) {
4039
_init_profile();
4140
if (nullptr == _sinker) {
4241
return Status::InternalError("sinker is NULL pointer.");
4342
}
4443
_is_dry_run = state->query_options().dry_run_query;
45-
// generate the arrow schema
46-
RETURN_IF_ERROR(convert_to_arrow_schema(_row_desc, &_arrow_schema));
4744
return Status::OK();
4845
}
4946

@@ -100,7 +97,7 @@ bool VArrowFlightResultWriter::can_sink() {
10097
return _sinker->can_sink();
10198
}
10299

103-
Status VArrowFlightResultWriter::close(Status) {
100+
Status VArrowFlightResultWriter::close(Status st) {
104101
COUNTER_SET(_sent_rows_counter, _written_rows);
105102
COUNTER_UPDATE(_bytes_sent_counter, _bytes_sent);
106103
return Status::OK();

‎be/src/vec/sink/varrow_flight_result_writer.h‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,15 +31,15 @@
3131
namespace doris {
3232
class BufferControlBlock;
3333
class RuntimeState;
34-
class RowDescriptor;
3534

3635
namespace vectorized {
3736
class Block;
3837

3938
class VArrowFlightResultWriter final : public ResultWriter {
4039
public:
4140
VArrowFlightResultWriter(BufferControlBlock* sinker, const VExprContextSPtrs& output_vexpr_ctxs,
42-
RuntimeProfile* parent_profile, const RowDescriptor& row_desc);
41+
RuntimeProfile* parent_profile,
42+
const std::shared_ptr<arrow::Schema>& arrow_schema);
4343

4444
Status init(RuntimeState* state) override;
4545

@@ -72,7 +72,6 @@ class VArrowFlightResultWriter final : public ResultWriter {
7272

7373
uint64_t _bytes_sent = 0;
7474

75-
const RowDescriptor& _row_desc;
7675
std::shared_ptr<arrow::Schema> _arrow_schema;
7776
};
7877
} // namespace vectorized

‎be/src/vec/sink/vmemory_scratch_sink.cpp‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,6 @@ Status MemoryScratchSink::_prepare_vexpr(RuntimeState* state) {
5656
RETURN_IF_ERROR(VExpr::create_expr_trees(_t_output_expr, _output_vexpr_ctxs));
5757
// Prepare the exprs to run.
5858
RETURN_IF_ERROR(VExpr::prepare(_output_vexpr_ctxs, state, _row_desc));
59-
// generate the arrow schema
60-
RETURN_IF_ERROR(convert_to_arrow_schema(_row_desc, &_arrow_schema));
6159
return Status::OK();
6260
}
6361

‎be/src/vec/sink/vmemory_scratch_sink.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,6 @@ class MemoryScratchSink final : public DataSink {
6565
private:
6666
Status _prepare_vexpr(RuntimeState* state);
6767

68-
std::shared_ptr<arrow::Schema> _arrow_schema;
69-
7068
BlockQueueSharedPtr _queue;
7169

7270
// Owned by the RuntimeState.

0 commit comments

Comments
 (0)