1818#include " format/arrow/arrow_row_batch.h"
1919
2020#include < arrow/buffer.h>
21+ #include < arrow/extension/parquet_variant.h>
2122#include < arrow/io/memory.h>
2223#include < arrow/ipc/writer.h>
2324#include < arrow/record_batch.h>
4849
4950namespace doris {
5051
52+ Status register_arrow_variant_extension () {
53+ // Remote Flight readers must restore the extension before decoding the result schema.
54+ static const auto status = [] {
55+ if (arrow::GetExtensionType (" arrow.parquet.variant" ) != nullptr ) {
56+ return arrow::Status::OK ();
57+ }
58+ return arrow::RegisterExtensionType (
59+ std::static_pointer_cast<arrow::ExtensionType>(arrow::extension::variant (
60+ arrow::struct_ ({arrow::field (" metadata" , arrow::binary (), false ),
61+ arrow::field (" value" , arrow::binary (), false )}))));
62+ }();
63+ return status.ok () ? Status::OK () : Status::InternalError (status.ToString ());
64+ }
65+
5166Status convert_to_arrow_type (const DataTypePtr& origin_type,
5267 std::shared_ptr<arrow::DataType>* result, const std::string& timezone,
5368 bool datetime_naive) {
69+ return convert_to_arrow_type (origin_type, result, timezone, datetime_naive, false );
70+ }
71+
72+ Status convert_to_arrow_type (const DataTypePtr& origin_type,
73+ std::shared_ptr<arrow::DataType>* result, const std::string& timezone,
74+ bool datetime_naive, bool native_variant) {
5475 auto type = get_serialized_type (origin_type);
5576 switch (type->get_primitive_type ()) {
5677 case TYPE_NULL :
@@ -137,7 +158,7 @@ Status convert_to_arrow_type(const DataTypePtr& origin_type,
137158 const auto * type_arr = assert_cast<const DataTypeArray*>(remove_nullable (type).get ());
138159 std::shared_ptr<arrow::DataType> item_type;
139160 RETURN_IF_ERROR (convert_to_arrow_type (type_arr->get_nested_type (), &item_type, timezone,
140- datetime_naive));
161+ datetime_naive, native_variant ));
141162 *result = std::make_shared<arrow::ListType>(item_type);
142163 break ;
143164 }
@@ -146,9 +167,9 @@ Status convert_to_arrow_type(const DataTypePtr& origin_type,
146167 std::shared_ptr<arrow::DataType> key_type;
147168 std::shared_ptr<arrow::DataType> val_type;
148169 RETURN_IF_ERROR (convert_to_arrow_type (type_map->get_key_type (), &key_type, timezone,
149- datetime_naive));
170+ datetime_naive, native_variant ));
150171 RETURN_IF_ERROR (convert_to_arrow_type (type_map->get_value_type (), &val_type, timezone,
151- datetime_naive));
172+ datetime_naive, native_variant ));
152173 *result = std::make_shared<arrow::MapType>(key_type, val_type);
153174 break ;
154175 }
@@ -158,7 +179,7 @@ Status convert_to_arrow_type(const DataTypePtr& origin_type,
158179 for (size_t i = 0 ; i < type_struct->get_elements ().size (); i++) {
159180 std::shared_ptr<arrow::DataType> field_type;
160181 RETURN_IF_ERROR (convert_to_arrow_type (type_struct->get_element (i), &field_type,
161- timezone, datetime_naive));
182+ timezone, datetime_naive, native_variant ));
162183 fields.push_back (
163184 std::make_shared<arrow::Field>(type_struct->get_element_name (i), field_type,
164185 type_struct->get_element (i)->is_nullable ()));
@@ -167,7 +188,13 @@ Status convert_to_arrow_type(const DataTypePtr& origin_type,
167188 break ;
168189 }
169190 case TYPE_VARIANT : {
170- *result = arrow::utf8 ();
191+ if (native_variant) {
192+ RETURN_IF_ERROR (register_arrow_variant_extension ());
193+ }
194+ *result = native_variant ? arrow::extension::variant (arrow::struct_ (
195+ {arrow::field (" metadata" , arrow::binary (), false ),
196+ arrow::field (" value" , arrow::binary (), false )}))
197+ : arrow::utf8 ();
171198 break ;
172199 }
173200 case TYPE_QUANTILE_STATE :
@@ -224,12 +251,20 @@ Status get_arrow_schema_from_block(const Block& block, std::shared_ptr<arrow::Sc
224251Status get_arrow_schema_from_expr_ctxs (const VExprContextSPtrs& output_vexpr_ctxs,
225252 std::shared_ptr<arrow::Schema>* result,
226253 const std::string& timezone, bool datetime_naive) {
254+ return get_arrow_schema_from_expr_ctxs (output_vexpr_ctxs, result, timezone, datetime_naive,
255+ false );
256+ }
257+
258+ Status get_arrow_schema_from_expr_ctxs (const VExprContextSPtrs& output_vexpr_ctxs,
259+ std::shared_ptr<arrow::Schema>* result,
260+ const std::string& timezone, bool datetime_naive,
261+ bool native_variant) {
227262 std::vector<std::shared_ptr<arrow::Field>> fields;
228263 for (int i = 0 ; i < output_vexpr_ctxs.size (); i++) {
229264 std::shared_ptr<arrow::DataType> arrow_type;
230265 auto root_expr = output_vexpr_ctxs.at (i)->root ();
231266 RETURN_IF_ERROR (convert_to_arrow_type (root_expr->data_type (), &arrow_type, timezone,
232- datetime_naive));
267+ datetime_naive, native_variant ));
233268 auto field_name = root_expr->is_slot_ref () && !root_expr->expr_label ().empty ()
234269 ? root_expr->expr_label ()
235270 : fmt::format (" {}_{}" , root_expr->data_type ()->get_name (), i);
0 commit comments