Skip to content

Commit 0b7e509

Browse files
committed
[improvement](be) Support Arrow loads and schema inference in FileScannerV2
1 parent 5964f9f commit 0b7e509

20 files changed

Lines changed: 1303 additions & 94 deletions

‎be/src/exec/operator/file_scan_operator.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,7 @@ bool FileScanLocalState::should_use_file_scanner_v2(const TQueryOptions& query_o
181181
case TFileFormatType::FORMAT_PROTO:
182182
case TFileFormatType::FORMAT_TEXT:
183183
case TFileFormatType::FORMAT_WAL:
184+
case TFileFormatType::FORMAT_ARROW:
184185
return true;
185186
default:
186187
return false;

‎be/src/exec/scan/file_scanner_v2.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -360,7 +360,10 @@ bool FileScannerV2::is_supported(const TFileScanRangeParams& params, const TFile
360360
} else if (format_type == TFileFormatType::FORMAT_ES_HTTP) {
361361
return table_format_name(range) == "es";
362362
} else if (format_type == TFileFormatType::FORMAT_ARROW) {
363-
return is_supported_arrow_table_format(range);
363+
// A plain Arrow range is an IPC load pipe; query Arrow ranges retain their connector.
364+
return is_supported_arrow_table_format(range) ||
365+
(table_format_name(range) == "NotSet" && params.__isset.src_tuple_id &&
366+
params.__isset.file_type && params.file_type == TFileType::FILE_STREAM);
364367
} else if (format_type == TFileFormatType::FORMAT_JNI) {
365368
return is_supported_jni_table_format(range);
366369
} else if (is_wal_format(format_type)) {
Lines changed: 347 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,347 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "format_v2/arrow/arrow_stream_reader.h"
19+
20+
#include <arrow/array.h>
21+
#include <arrow/buffer.h>
22+
#include <arrow/compute/api.h>
23+
#include <arrow/io/interfaces.h>
24+
#include <arrow/ipc/reader.h>
25+
#include <arrow/record_batch.h>
26+
#include <arrow/type.h>
27+
28+
#include <utility>
29+
30+
#include "core/block/block.h"
31+
#include "format/arrow/arrow_array_normalizer.h"
32+
#include "format/arrow/arrow_row_batch.h"
33+
#include "format/arrow/arrow_utils.h"
34+
#include "format/parquet/arrow_memory_pool.h"
35+
#include "format_v2/column_mapper.h"
36+
#include "format_v2/materialized_reader_util.h"
37+
#include "io/fs/tracing_file_reader.h"
38+
#include "runtime/descriptors.h"
39+
#include "runtime/file_scan_profile.h"
40+
#include "runtime/runtime_state.h"
41+
42+
namespace doris::format::arrow_stream {
43+
44+
namespace {
45+
46+
std::shared_ptr<arrow::DataType> input_binding_type(
47+
const std::shared_ptr<arrow::DataType>& wire,
48+
const std::shared_ptr<arrow::DataType>& declared) {
49+
if (wire->Equals(declared)) {
50+
return declared;
51+
}
52+
// SerDe owns datetime rounding and naive/aware interpretation. Casting timestamp[us]
53+
// to the declared timestamp[ms] first would reject .123789 instead of rounding to .124.
54+
if (wire->id() == arrow::Type::TIMESTAMP && declared->id() == arrow::Type::TIMESTAMP) {
55+
return wire;
56+
}
57+
if (declared->id() == arrow::Type::LIST &&
58+
(wire->id() == arrow::Type::LIST || wire->id() == arrow::Type::LARGE_LIST ||
59+
wire->id() == arrow::Type::FIXED_SIZE_LIST)) {
60+
const auto& source = static_cast<const arrow::BaseListType&>(*wire);
61+
const auto& target = static_cast<const arrow::ListType&>(*declared);
62+
return arrow::list(target.value_field()->WithType(
63+
input_binding_type(source.value_type(), target.value_type())));
64+
}
65+
if (wire->id() == arrow::Type::MAP && declared->id() == arrow::Type::MAP) {
66+
const auto& source = static_cast<const arrow::MapType&>(*wire);
67+
const auto& target = static_cast<const arrow::MapType&>(*declared);
68+
return std::make_shared<arrow::MapType>(target.key_field()->WithType(input_binding_type(
69+
source.key_type(), target.key_type())),
70+
target.item_field()->WithType(input_binding_type(
71+
source.item_type(), target.item_type())),
72+
target.keys_sorted());
73+
}
74+
if (wire->id() == arrow::Type::STRUCT && declared->id() == arrow::Type::STRUCT &&
75+
wire->num_fields() == declared->num_fields()) {
76+
arrow::FieldVector fields;
77+
for (int i = 0; i < wire->num_fields(); ++i) {
78+
fields.push_back(declared->field(i)->WithType(
79+
input_binding_type(wire->field(i)->type(), declared->field(i)->type())));
80+
}
81+
return arrow::struct_(fields);
82+
}
83+
return declared;
84+
}
85+
86+
} // namespace
87+
88+
// Peek one byte between IPC streams without requesting a large buffer from a blocking load pipe.
89+
// For example, a second schema can immediately follow the first stream's EOS marker.
90+
class ArrowInputStream final : public arrow::io::InputStream {
91+
public:
92+
ArrowInputStream(io::FileReaderSPtr reader, std::shared_ptr<io::IOContext> io_ctx,
93+
arrow::MemoryPool* pool)
94+
: _reader(std::move(reader)), _io_ctx(std::move(io_ctx)), _pool(pool) {}
95+
96+
arrow::Status Close() override {
97+
_reader.reset();
98+
return arrow::Status::OK();
99+
}
100+
bool closed() const override { return _reader == nullptr; }
101+
arrow::Result<int64_t> Tell() const override { return _position; }
102+
103+
Status has_next(bool* has_next) {
104+
size_t bytes = 0;
105+
RETURN_IF_ERROR(_reader->read_at(_position, Slice(&_next_byte, 1), &bytes, _io_ctx.get()));
106+
_has_next_byte = bytes != 0;
107+
*has_next = _has_next_byte;
108+
return Status::OK();
109+
}
110+
111+
arrow::Result<int64_t> Read(int64_t nbytes, void* out) override {
112+
auto* output = static_cast<char*>(out);
113+
int64_t total = 0;
114+
if (nbytes > 0 && _has_next_byte) {
115+
output[total++] = _next_byte;
116+
++_position;
117+
_has_next_byte = false;
118+
}
119+
while (total < nbytes) {
120+
size_t bytes = 0;
121+
const auto status = _reader->read_at(_position, Slice(output + total, nbytes - total),
122+
&bytes, _io_ctx.get());
123+
if (!status.ok()) {
124+
return arrow::Status::IOError(status.to_string());
125+
}
126+
if (bytes == 0) {
127+
break;
128+
}
129+
total += bytes;
130+
_position += bytes;
131+
}
132+
return total;
133+
}
134+
arrow::Result<std::shared_ptr<arrow::Buffer>> Read(int64_t nbytes) override {
135+
ARROW_ASSIGN_OR_RAISE(auto buffer, arrow::AllocateResizableBuffer(nbytes, _pool));
136+
ARROW_ASSIGN_OR_RAISE(auto bytes, Read(nbytes, buffer->mutable_data()));
137+
ARROW_RETURN_NOT_OK(buffer->Resize(bytes, false));
138+
buffer->ZeroPadding();
139+
return buffer;
140+
}
141+
142+
private:
143+
io::FileReaderSPtr _reader;
144+
std::shared_ptr<io::IOContext> _io_ctx;
145+
arrow::MemoryPool* _pool;
146+
int64_t _position = 0;
147+
char _next_byte = 0;
148+
bool _has_next_byte = false;
149+
};
150+
151+
ArrowStreamReader::ArrowStreamReader(std::shared_ptr<io::FileSystemProperties>& properties,
152+
std::unique_ptr<io::FileDescription>& file,
153+
std::shared_ptr<io::IOContext> io_ctx, RuntimeProfile* profile,
154+
std::vector<SlotDescriptor*> file_slots,
155+
std::optional<TUniqueId> load_id)
156+
: FileReader(properties, file, std::move(io_ctx), profile),
157+
_file_slots(std::move(file_slots)),
158+
_load_id(std::move(load_id)),
159+
_arrow_pool(std::make_unique<ArrowMemoryPool<>>()) {}
160+
161+
ArrowStreamReader::~ArrowStreamReader() {
162+
static_cast<void>(close());
163+
}
164+
165+
void ArrowStreamReader::_init_profile() {
166+
if (_profile == nullptr) {
167+
return;
168+
}
169+
file_scan_profile::ensure_hierarchy(_profile);
170+
_total_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ArrowStreamReader",
171+
file_scan_profile::FILE_READER, 1);
172+
_read_time =
173+
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ArrowStreamReadTime", "ArrowStreamReader", 1);
174+
_materialize_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ArrowStreamMaterializeTime",
175+
"ArrowStreamReader", 1);
176+
}
177+
178+
Status ArrowStreamReader::init(RuntimeState* state) {
179+
_state = state;
180+
_init_profile();
181+
SCOPED_TIMER(_total_time);
182+
if (_system_properties->system_type != TFileType::FILE_STREAM || !_load_id) {
183+
return Status::InvalidArgument("Arrow IPC load requires a stream load id");
184+
}
185+
for (const auto* slot : _file_slots) {
186+
std::shared_ptr<arrow::DataType> type;
187+
RETURN_IF_ERROR(DorisArrowSchemaConvertor(state->timezone())
188+
.convert_to_arrow_type(slot->type(), &type));
189+
_arrow_types.push_back(std::move(type));
190+
}
191+
_eof = false;
192+
return Status::OK();
193+
}
194+
195+
Status ArrowStreamReader::get_schema(std::vector<ColumnDefinition>* schema) const {
196+
schema->clear();
197+
for (size_t i = 0; i < _file_slots.size(); ++i) {
198+
const auto* slot = _file_slots[i];
199+
schema->push_back({.identifier = Field::create_field<TYPE_STRING>(slot->col_name()),
200+
.local_id = cast_set<int32_t>(i),
201+
.name = slot->col_name(),
202+
.type = slot->type(),
203+
.children = materialized_schema_children(slot->type())});
204+
}
205+
return Status::OK();
206+
}
207+
208+
std::unique_ptr<TableColumnMapper> ArrowStreamReader::create_column_mapper(
209+
TableColumnMapperOptions options) const {
210+
return std::make_unique<MaterializedColumnMapper>(options);
211+
}
212+
213+
Status ArrowStreamReader::open(std::shared_ptr<FileScanRequest> request) {
214+
RETURN_IF_ERROR(FileReader::open(std::move(request)));
215+
RETURN_IF_ERROR(FileFactory::create_pipe_reader(*_load_id, &_file_reader, _state, false));
216+
_tracing_file_reader = _io_ctx && _io_ctx->file_reader_stats
217+
? std::make_shared<io::TracingFileReader>(
218+
_file_reader, _io_ctx->file_reader_stats)
219+
: _file_reader;
220+
_input = std::make_shared<ArrowInputStream>(_tracing_file_reader, _io_ctx, _arrow_pool.get());
221+
return Status::OK();
222+
}
223+
224+
Status ArrowStreamReader::_open_next_stream() {
225+
bool has_next = false;
226+
RETURN_IF_ERROR(_input->has_next(&has_next));
227+
if (!has_next) {
228+
_eof = true;
229+
return Status::OK();
230+
}
231+
auto options = arrow::ipc::IpcReadOptions::Defaults();
232+
options.memory_pool = _arrow_pool.get();
233+
// Keep IPC decompression on the scanner thread, where the load memory tracker is attached.
234+
options.use_threads = false;
235+
RETURN_DORIS_STATUS_IF_RESULT_ERROR(_ipc_reader,
236+
arrow::ipc::RecordBatchStreamReader::Open(_input, options));
237+
_arrow_positions.assign(_file_slots.size(), -1);
238+
for (const auto& [id, position] : _request->local_positions) {
239+
const auto& name = _file_slots.at(id.value())->col_name();
240+
const auto indices = _ipc_reader->schema()->GetAllFieldIndices(name);
241+
if (indices.size() != 1) {
242+
return Status::InvalidArgument("Arrow stream requires exactly one column named '{}'",
243+
name);
244+
}
245+
_arrow_positions[id.value()] = indices[0];
246+
}
247+
return Status::OK();
248+
}
249+
250+
Status ArrowStreamReader::_prepare_arrays() {
251+
_arrays.assign(_file_slots.size(), nullptr);
252+
for (const auto& [id, position] : _request->local_positions) {
253+
std::shared_ptr<arrow::Array> array;
254+
RETURN_IF_ERROR(normalize_arrow_array(_batch->column(_arrow_positions[id.value()]),
255+
_arrow_pool.get(), &array));
256+
const auto binding = input_binding_type(array->type(), _arrow_types[id.value()]);
257+
// Bind differing numeric widths before concrete SerDe array casts. Timestamp units
258+
// stay on the wire even inside containers, so Doris retains its own rounding semantics.
259+
if (!array->type()->Equals(binding)) {
260+
arrow::compute::ExecContext context(_arrow_pool.get());
261+
auto options = arrow::compute::CastOptions::Safe();
262+
// The Arrow load protocol permits binary payloads in Doris STRING columns.
263+
options.allow_invalid_utf8 = true;
264+
auto converted = arrow::compute::Cast(array, binding, options, &context);
265+
if (!converted.ok()) {
266+
return Status::InvalidArgument("Cannot decode Arrow column '{}': {}",
267+
_file_slots[id.value()]->col_name(),
268+
converted.status().ToString());
269+
}
270+
array = converted->make_array();
271+
}
272+
_arrays[id.value()] = std::move(array);
273+
}
274+
return Status::OK();
275+
}
276+
277+
Status ArrowStreamReader::_next_batch() {
278+
SCOPED_TIMER(_read_time);
279+
while (!_batch || _batch_offset == _batch->num_rows()) {
280+
_batch.reset();
281+
_arrays.clear();
282+
_batch_offset = 0;
283+
if (!_ipc_reader) {
284+
RETURN_IF_ERROR(_open_next_stream());
285+
if (_eof) {
286+
return Status::OK();
287+
}
288+
}
289+
RETURN_DORIS_STATUS_IF_ERROR(_ipc_reader->ReadNext(&_batch));
290+
if (!_batch) {
291+
_ipc_reader.reset();
292+
continue;
293+
}
294+
RETURN_DORIS_STATUS_IF_ERROR(_batch->Validate());
295+
if (_batch->num_rows() != 0) {
296+
RETURN_IF_ERROR(_prepare_arrays());
297+
}
298+
}
299+
return Status::OK();
300+
}
301+
302+
Status ArrowStreamReader::get_block(Block* block, size_t* rows, bool* eof) {
303+
SCOPED_TIMER(_total_time);
304+
*rows = 0;
305+
*eof = false;
306+
if (_eof || (_io_ctx && _io_ctx->should_stop)) {
307+
*eof = true;
308+
return Status::OK();
309+
}
310+
RETURN_IF_ERROR(_next_batch());
311+
if (_eof) {
312+
*eof = true;
313+
return Status::OK();
314+
}
315+
const int64_t end = _batch_offset +
316+
std::min<int64_t>(_state->batch_size(), _batch->num_rows() - _batch_offset);
317+
{
318+
SCOPED_TIMER(_materialize_time);
319+
auto guard = block->mutate_columns_scoped();
320+
auto& columns = guard.mutable_columns();
321+
for (const auto& [id, position] : _request->local_positions) {
322+
// Preserve the original array and pass a row range: Arrow Slice creates nonzero
323+
// offsets, which nested Doris SerDes deliberately reject.
324+
RETURN_IF_ERROR(guard.get_datatype_by_position(position.value())
325+
->get_serde()
326+
->read_column_from_arrow(*columns.at(position.value()),
327+
_arrays[id.value()].get(),
328+
_batch_offset, end,
329+
_state->timezone_obj()));
330+
}
331+
}
332+
*rows = cast_set<size_t>(end - _batch_offset);
333+
_batch_offset = end;
334+
_record_scan_rows(cast_set<int64_t>(*rows));
335+
return apply_materialized_reader_filters(_request.get(), _io_ctx.get(), block, rows);
336+
}
337+
338+
Status ArrowStreamReader::close() {
339+
_batch.reset();
340+
_arrays.clear();
341+
_ipc_reader.reset();
342+
_input.reset();
343+
_request.reset();
344+
return FileReader::close();
345+
}
346+
347+
} // namespace doris::format::arrow_stream

0 commit comments

Comments
 (0)