Skip to content

Commit 3981f52

Browse files
committed
[feature](flight) Add opt-in native Variant results for ADBC
1 parent 7b50e05 commit 3981f52

18 files changed

Lines changed: 790 additions & 29 deletions

File tree

‎be/src/core/data_type_serde/data_type_variant_serde.cpp‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include <arrow/array/builder_binary.h>
2121

2222
#include <cstdint>
23+
#include <limits>
2324
#include <string>
2425

2526
#include "common/cast_set.h"
@@ -29,12 +30,17 @@
2930
#include "core/assert_cast.h"
3031
#include "core/column/column.h"
3132
#include "core/column/column_variant.h"
33+
#include "core/column/variant_v2/column_variant_v2.h"
34+
#include "core/column/variant_v2/column_variant_v2_typed_column.h"
35+
#include "core/data_type/data_type_nullable.h"
3236
#include "core/data_type_serde/data_type_serde.h"
37+
#include "core/data_type_serde/data_type_variant_v2_serde.h"
3338
#include "core/field.h"
3439
#include "core/string_ref.h"
3540
#include "core/types.h"
3641
#include "core/value/jsonb_value.h"
3742
#include "exec/common/variant_util.h"
43+
#include "exprs/function/parse/variant_string_parse.h"
3844
#include "util/json/json_parser.h"
3945
#include "util/jsonb_writer.h"
4046

@@ -157,6 +163,64 @@ Status DataTypeVariantSerDe::write_column_to_arrow(const IColumn& column, const
157163
int64_t start, int64_t end,
158164
const cctz::time_zone& ctz) const {
159165
const auto* var = check_and_get_column<ColumnVariant>(column);
166+
if (array_builder->type()->id() == arrow::Type::STRUCT) {
167+
// Legacy documents need JSON conversion; typed scalar roots can keep their type.
168+
// The outer null map must remain SQL NULL on the wire.
169+
if (start < 0 || end < start || end > column.size() ||
170+
(null_map != nullptr && end > null_map->size())) {
171+
return Status::InvalidArgument("Invalid Variant Arrow row range [{}, {})", start, end);
172+
}
173+
if (var->is_scalar_variant()) {
174+
auto scalar_type = remove_nullable(var->get_root_type());
175+
if (scalar_type->get_primitive_type() == TYPE_DECIMAL256) {
176+
return Status::NotSupported(
177+
"Native Arrow Variant does not support Decimal256 roots");
178+
}
179+
if (is_supported_variant_typed_identity(scalar_type->get_primitive_type())) {
180+
// Avoid a JSON round trip that would turn exact decimal roots into doubles.
181+
auto typed =
182+
ColumnVariantV2::create_typed(make_nullable(var->get_root()), scalar_type);
183+
return DataTypeVariantV2SerDe().write_column_to_arrow(
184+
*typed, null_map, array_builder, start, end, ctz);
185+
}
186+
}
187+
JsonToVariantOptions parse_options;
188+
parse_options.throw_on_invalid_json = true;
189+
// Stored keys were already accepted at ingestion; mutable parse limits must not reject reads.
190+
parse_options.max_json_key_length = std::numeric_limits<uint32_t>::max();
191+
parse_options.check_duplicate_json_path = false;
192+
JsonStringToVariantEncoder encoder(parse_options);
193+
FormatOptions options;
194+
options.timezone = &ctz;
195+
NullMap selected_nulls;
196+
if (null_map != nullptr) {
197+
selected_nulls.assign(null_map->begin() + start, null_map->begin() + end);
198+
}
199+
for (int64_t row = start; row < end; ++row) {
200+
std::string json;
201+
if (null_map != nullptr && (*null_map)[row]) {
202+
json = "null";
203+
} else {
204+
var->serialize_one_row_to_string(row, &json, options);
205+
if (var->get_root_type()->get_primitive_type() == TYPE_STRING &&
206+
!var->get_root()->is_null_at(row)) {
207+
// The legacy root string serializer emits raw text, not a JSON string literal.
208+
auto quoted = ColumnString::create();
209+
VectorBufferWriter writer(*quoted);
210+
writer.write_json_string(json);
211+
writer.commit();
212+
json = quoted->get_data_at(0).to_string();
213+
}
214+
}
215+
encoder.add_json({json.data(), json.size()});
216+
}
217+
auto values = encoder.finish_batch();
218+
auto encoded = ColumnVariantV2::create();
219+
encoded->insert_encoded_batch(values);
220+
return DataTypeVariantV2SerDe().write_column_to_arrow(
221+
*encoded, null_map == nullptr ? nullptr : &selected_nulls, array_builder, 0,
222+
end - start, ctz);
223+
}
160224
if (array_builder->type()->id() == arrow::Type::LARGE_STRING) {
161225
auto& builder = assert_cast<arrow::LargeStringBuilder&>(*array_builder);
162226
return write_variant_column_to_arrow_impl(column, *var, null_map, builder, start, end, ctz);

‎be/src/core/data_type_serde/data_type_variant_v2_serde.cpp‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -625,13 +625,13 @@ Status write_paimon_variant(const IColumn& column, const NullMap* null_map,
625625
return status;
626626
}
627627

628-
Status write_iceberg_variant(const IColumn& column, const NullMap* null_map,
629-
arrow::ArrayBuilder* array_builder, int64_t start, int64_t end) {
628+
Status write_parquet_variant_arrow(const IColumn& column, const NullMap* null_map,
629+
arrow::ArrayBuilder* array_builder, int64_t start, int64_t end) {
630630
if (start < 0 || end < start) {
631-
return Status::InvalidArgument("Invalid Iceberg Variant row range [{}, {})", start, end);
631+
return Status::InvalidArgument("Invalid Variant Arrow row range [{}, {})", start, end);
632632
}
633633
if (array_builder->type()->id() != arrow::Type::STRUCT) {
634-
return Status::InvalidArgument("Iceberg Variant writer requires a struct builder, got {}",
634+
return Status::InvalidArgument("Variant Arrow writer requires a struct builder, got {}",
635635
array_builder->type()->ToString());
636636
}
637637
auto& builder = assert_cast<arrow::StructBuilder&>(*array_builder);
@@ -640,7 +640,7 @@ Status write_iceberg_variant(const IColumn& column, const NullMap* null_map,
640640
type->field(1)->name() != "value" || type->field(0)->type()->id() != arrow::Type::BINARY ||
641641
type->field(1)->type()->id() != arrow::Type::BINARY) {
642642
return Status::InvalidArgument(
643-
"Iceberg Variant writer requires struct<metadata: binary, value: binary>, got {}",
643+
"Variant Arrow writer requires struct<metadata: binary, value: binary>, got {}",
644644
type->ToString());
645645
}
646646
auto& metadata_builder = assert_cast<arrow::BinaryBuilder&>(*builder.field_builder(0));
@@ -660,7 +660,7 @@ Status write_iceberg_variant(const IColumn& column, const NullMap* null_map,
660660
if (value.metadata.size > std::numeric_limits<int32_t>::max() ||
661661
value.value.size > std::numeric_limits<int32_t>::max()) {
662662
status = Status::InvalidArgument(
663-
"Iceberg Variant metadata/value exceeds Arrow binary size limit");
663+
"Variant Arrow metadata/value exceeds Arrow binary size limit");
664664
return;
665665
}
666666
status = checkArrowStatus(builder.Append(), column, builder);
@@ -733,6 +733,9 @@ Status DataTypeVariantV2SerDe::write_column_to_arrow(const IColumn& column, cons
733733
options.timezone = &ctz;
734734
const size_t first = checked_row(start);
735735
const size_t last = checked_row(end);
736+
if (array_builder->type()->id() == arrow::Type::STRUCT) {
737+
return write_parquet_variant_arrow(column, null_map, array_builder, start, end);
738+
}
736739
if (array_builder->type()->id() == arrow::Type::STRING) {
737740
return write_arrow(column, null_map, assert_cast<arrow::StringBuilder&>(*array_builder),
738741
first, last, options);
@@ -759,7 +762,7 @@ Status DataTypeVariantV2SerDe::write_column_to_iceberg_arrow(
759762
const std::shared_ptr<const IDataType>&, const IColumn& column, const NullMap* null_map,
760763
const std::shared_ptr<arrow::Field>&, arrow::ArrayBuilder* array_builder, int64_t start,
761764
int64_t end, const cctz::time_zone&) const {
762-
return write_iceberg_variant(column, null_map, array_builder, start, end);
765+
return write_parquet_variant_arrow(column, null_map, array_builder, start, end);
763766
}
764767

765768
Status DataTypeVariantV2SerDe::write_column_to_orc(const std::string&, const IColumn& column,

‎be/src/exec/operator/result_sink_operator.cpp‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,9 @@ Status ResultSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info)
5757
} else {
5858
std::shared_ptr<arrow::Schema> arrow_schema;
5959
if (p._sink_type == TResultSinkType::ARROW_FLIGHT_PROTOCOL) {
60-
RETURN_IF_ERROR(get_arrow_schema_from_expr_ctxs(_output_vexpr_ctxs, &arrow_schema,
61-
state->timezone(),
62-
/*datetime_naive=*/true));
60+
RETURN_IF_ERROR(get_arrow_schema_from_expr_ctxs(
61+
_output_vexpr_ctxs, &arrow_schema, state->timezone(),
62+
/*datetime_naive=*/true, p._native_variant));
6363
}
6464
VLOG_DEBUG << "create sender in INIT with instance id " << fragment_instance_id;
6565
RETURN_IF_ERROR(state->exec_env()->result_mgr()->create_sender(
@@ -103,6 +103,7 @@ ResultSinkOperatorX::ResultSinkOperatorX(int operator_id, int node_id,
103103
_sink_type(!sink.__isset.type || sink.type == TResultSinkType::MYSQL_PROTOCOL
104104
? TResultSinkType::MYSQL_PROTOCOL
105105
: sink.type),
106+
_native_variant(sink.native_variant),
106107
_result_sink_buffer_size_rows(_sink_type == TResultSinkType::ARROW_FLIGHT_PROTOCOL
107108
? config::arrow_flight_result_sink_buffer_size_rows
108109
: RESULT_SINK_BUFFER_SIZE),
@@ -123,9 +124,9 @@ Status ResultSinkOperatorX::prepare(RuntimeState* state) {
123124
if (state->query_options().enable_parallel_result_sink) {
124125
std::shared_ptr<arrow::Schema> arrow_schema;
125126
if (_sink_type == TResultSinkType::ARROW_FLIGHT_PROTOCOL) {
126-
RETURN_IF_ERROR(get_arrow_schema_from_expr_ctxs(_output_vexpr_ctxs, &arrow_schema,
127-
state->timezone(),
128-
/*datetime_naive=*/true));
127+
RETURN_IF_ERROR(get_arrow_schema_from_expr_ctxs(
128+
_output_vexpr_ctxs, &arrow_schema, state->timezone(),
129+
/*datetime_naive=*/true, _native_variant));
129130
}
130131
VLOG_DEBUG << "create sender in prepare with query id " << state->query_id();
131132
RETURN_IF_ERROR(state->exec_env()->result_mgr()->create_sender(

‎be/src/exec/operator/result_sink_operator.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,7 @@ class ResultSinkOperatorX final : public DataSinkOperatorX<ResultSinkLocalState>
167167

168168
Status _second_phase_fetch_data(RuntimeState* state, Block* final_block);
169169
const TResultSinkType::type _sink_type;
170+
const bool _native_variant;
170171
const int _result_sink_buffer_size_rows;
171172
// set file options when sink type is FILE
172173
std::unique_ptr<ResultFileOptions> _file_opts = nullptr;

‎be/src/format/arrow/arrow_block_convertor.cpp‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -472,6 +472,26 @@ Status ArrowBlockConvertor::init() {
472472
return Status::OK();
473473
}
474474

475+
Status ArrowFlightArrowBlockConvertor::write_column(const std::shared_ptr<const IDataType>& type,
476+
const DataTypeSerDe& serde,
477+
const IColumn& column, const NullMap* null_map,
478+
const std::shared_ptr<arrow::Field>& field,
479+
arrow::ArrayBuilder* array_builder,
480+
int64_t start, int64_t end,
481+
const cctz::time_zone& ctz) const {
482+
if (contains_extension_type(field->type())) {
483+
std::shared_ptr<arrow::DataType> native_type;
484+
RETURN_IF_ERROR(convert_to_arrow_type(type, &native_type, ctz.name(), true, true));
485+
// Check the extension identity and its complete nested shape before allowing the
486+
// Variant SerDe to write binary storage. An arbitrary STRUCT is not a Variant binding.
487+
if (native_type->Equals(field->type())) {
488+
return serde.write_column_to_arrow(column, null_map, array_builder, start, end, ctz);
489+
}
490+
}
491+
return DorisArrowBlockConvertor::write_column(type, serde, column, null_map, field,
492+
array_builder, start, end, ctz);
493+
}
494+
475495
Status ArrowFlightArrowBlockConvertor::convert_to_arrow(const Block& block, arrow::MemoryPool* pool,
476496
std::shared_ptr<arrow::RecordBatch>* result,
477497
size_t start_row, size_t end_row) const {

‎be/src/format/arrow/arrow_block_convertor.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,13 @@ class ArrowFlightArrowBlockConvertor final : public DorisArrowBlockConvertor {
119119
Status convert_to_arrow(const Block& block, arrow::MemoryPool* pool,
120120
std::shared_ptr<arrow::RecordBatch>* result, size_t start_row = 0,
121121
size_t end_row = 0) const override;
122+
123+
protected:
124+
Status write_column(const std::shared_ptr<const IDataType>& type, const DataTypeSerDe& serde,
125+
const IColumn& column, const NullMap* null_map,
126+
const std::shared_ptr<arrow::Field>& field,
127+
arrow::ArrayBuilder* array_builder, int64_t start, int64_t end,
128+
const cctz::time_zone& ctz) const override;
122129
};
123130

124131
class PythonArrowBlockConvertor final : public DorisArrowBlockConvertor {

‎be/src/format/arrow/arrow_row_batch.cpp‎

Lines changed: 41 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include "format/arrow/arrow_row_batch.h"
1919

2020
#include <arrow/buffer.h>
21+
#include <arrow/extension/parquet_variant.h>
2122
#include <arrow/io/memory.h>
2223
#include <arrow/ipc/writer.h>
2324
#include <arrow/record_batch.h>
@@ -48,9 +49,29 @@
4849

4950
namespace doris {
5051

52+
Status register_arrow_variant_extension() {
53+
// Remote Flight readers must restore the extension before decoding the result schema.
54+
static const auto status = [] {
55+
if (arrow::GetExtensionType("arrow.parquet.variant") != nullptr) {
56+
return arrow::Status::OK();
57+
}
58+
return arrow::RegisterExtensionType(
59+
std::static_pointer_cast<arrow::ExtensionType>(arrow::extension::variant(
60+
arrow::struct_({arrow::field("metadata", arrow::binary(), false),
61+
arrow::field("value", arrow::binary(), false)}))));
62+
}();
63+
return status.ok() ? Status::OK() : Status::InternalError(status.ToString());
64+
}
65+
5166
Status convert_to_arrow_type(const DataTypePtr& origin_type,
5267
std::shared_ptr<arrow::DataType>* result, const std::string& timezone,
5368
bool datetime_naive) {
69+
return convert_to_arrow_type(origin_type, result, timezone, datetime_naive, false);
70+
}
71+
72+
Status convert_to_arrow_type(const DataTypePtr& origin_type,
73+
std::shared_ptr<arrow::DataType>* result, const std::string& timezone,
74+
bool datetime_naive, bool native_variant) {
5475
auto type = get_serialized_type(origin_type);
5576
switch (type->get_primitive_type()) {
5677
case TYPE_NULL:
@@ -137,7 +158,7 @@ Status convert_to_arrow_type(const DataTypePtr& origin_type,
137158
const auto* type_arr = assert_cast<const DataTypeArray*>(remove_nullable(type).get());
138159
std::shared_ptr<arrow::DataType> item_type;
139160
RETURN_IF_ERROR(convert_to_arrow_type(type_arr->get_nested_type(), &item_type, timezone,
140-
datetime_naive));
161+
datetime_naive, native_variant));
141162
*result = std::make_shared<arrow::ListType>(item_type);
142163
break;
143164
}
@@ -146,9 +167,9 @@ Status convert_to_arrow_type(const DataTypePtr& origin_type,
146167
std::shared_ptr<arrow::DataType> key_type;
147168
std::shared_ptr<arrow::DataType> val_type;
148169
RETURN_IF_ERROR(convert_to_arrow_type(type_map->get_key_type(), &key_type, timezone,
149-
datetime_naive));
170+
datetime_naive, native_variant));
150171
RETURN_IF_ERROR(convert_to_arrow_type(type_map->get_value_type(), &val_type, timezone,
151-
datetime_naive));
172+
datetime_naive, native_variant));
152173
*result = std::make_shared<arrow::MapType>(key_type, val_type);
153174
break;
154175
}
@@ -158,7 +179,7 @@ Status convert_to_arrow_type(const DataTypePtr& origin_type,
158179
for (size_t i = 0; i < type_struct->get_elements().size(); i++) {
159180
std::shared_ptr<arrow::DataType> field_type;
160181
RETURN_IF_ERROR(convert_to_arrow_type(type_struct->get_element(i), &field_type,
161-
timezone, datetime_naive));
182+
timezone, datetime_naive, native_variant));
162183
fields.push_back(
163184
std::make_shared<arrow::Field>(type_struct->get_element_name(i), field_type,
164185
type_struct->get_element(i)->is_nullable()));
@@ -167,7 +188,13 @@ Status convert_to_arrow_type(const DataTypePtr& origin_type,
167188
break;
168189
}
169190
case TYPE_VARIANT: {
170-
*result = arrow::utf8();
191+
if (native_variant) {
192+
RETURN_IF_ERROR(register_arrow_variant_extension());
193+
}
194+
*result = native_variant ? arrow::extension::variant(arrow::struct_(
195+
{arrow::field("metadata", arrow::binary(), false),
196+
arrow::field("value", arrow::binary(), false)}))
197+
: arrow::utf8();
171198
break;
172199
}
173200
case TYPE_QUANTILE_STATE:
@@ -224,12 +251,20 @@ Status get_arrow_schema_from_block(const Block& block, std::shared_ptr<arrow::Sc
224251
Status get_arrow_schema_from_expr_ctxs(const VExprContextSPtrs& output_vexpr_ctxs,
225252
std::shared_ptr<arrow::Schema>* result,
226253
const std::string& timezone, bool datetime_naive) {
254+
return get_arrow_schema_from_expr_ctxs(output_vexpr_ctxs, result, timezone, datetime_naive,
255+
false);
256+
}
257+
258+
Status get_arrow_schema_from_expr_ctxs(const VExprContextSPtrs& output_vexpr_ctxs,
259+
std::shared_ptr<arrow::Schema>* result,
260+
const std::string& timezone, bool datetime_naive,
261+
bool native_variant) {
227262
std::vector<std::shared_ptr<arrow::Field>> fields;
228263
for (int i = 0; i < output_vexpr_ctxs.size(); i++) {
229264
std::shared_ptr<arrow::DataType> arrow_type;
230265
auto root_expr = output_vexpr_ctxs.at(i)->root();
231266
RETURN_IF_ERROR(convert_to_arrow_type(root_expr->data_type(), &arrow_type, timezone,
232-
datetime_naive));
267+
datetime_naive, native_variant));
233268
auto field_name = root_expr->is_slot_ref() && !root_expr->expr_label().empty()
234269
? root_expr->expr_label()
235270
: fmt::format("{}_{}", root_expr->data_type()->get_name(), i);

0 commit comments

Comments
 (0)