2929#include < utility>
3030
3131#include " core/block/block.h"
32+ #include " core/column/column_fixed_length_object.h"
33+ #include " core/data_type/data_type_agg_state.h"
3234#include " core/data_type/data_type_array.h"
3335#include " core/data_type/data_type_map.h"
3436#include " core/data_type/data_type_nullable.h"
@@ -48,6 +50,37 @@ namespace doris::format::arrow_stream {
4850
4951namespace {
5052
53+ class ArrowLoadSchemaConvertor final : public DorisArrowSchemaConvertor {
54+ public:
55+ using DorisArrowSchemaConvertor::DorisArrowSchemaConvertor;
56+
57+ Status convert_to_arrow_type (const DataTypePtr& origin_type,
58+ std::shared_ptr<arrow::DataType>* result) const override {
59+ const auto type = remove_nullable (origin_type);
60+ // VARIANT output may be UTF-8, but its input SerDe requires the native binary carrier,
61+ // including when VARIANT is nested in an ARRAY, MAP, or STRUCT.
62+ if (type->get_primitive_type () == TYPE_VARIANT ) {
63+ *result = arrow::struct_ ({arrow::field (" metadata" , arrow::binary (), false ),
64+ arrow::field (" value" , arrow::binary (), false )});
65+ return Status::OK ();
66+ }
67+ if (type->get_primitive_type () == TYPE_AGG_STATE ) {
68+ const auto & state = assert_cast<const DataTypeAggState&>(*type);
69+ const auto serialized_type = remove_nullable (state.get_serialized_type ());
70+ // FixedLengthObject's type omits its width. The serialization column owns that
71+ // contract (e.g. count(INT NOT NULL) uses eight bytes), not the runtime state size.
72+ if (serialized_type->get_primitive_type () == TYPE_FIXED_LENGTH_OBJECT ) {
73+ const auto column = state.create_column ();
74+ const auto width = assert_cast<const ColumnFixedLengthObject&>(*column).item_size ();
75+ *result = arrow::fixed_size_binary (cast_set<int32_t >(width));
76+ return Status::OK ();
77+ }
78+ return convert_to_arrow_type (serialized_type, result);
79+ }
80+ return DorisArrowSchemaConvertor::convert_to_arrow_type (type, result);
81+ }
82+ };
83+
5184arrow::Result<std::shared_ptr<arrow::DataType>> input_binding_type (
5285 const std::shared_ptr<arrow::DataType>& wire,
5386 const std::shared_ptr<arrow::DataType>& declared, const DataTypePtr& doris_type) {
@@ -63,7 +96,12 @@ arrow::Result<std::shared_ptr<arrow::DataType>> input_binding_type(
6396 if (wire->id () == arrow::Type::DATE64 && declared->id () == arrow::Type::DATE32 ) {
6497 return wire;
6598 }
66- const auto type = remove_nullable (doris_type);
99+ const auto type = remove_nullable (get_serialized_type (remove_nullable (doris_type)));
100+ // A native VARIANT carrier is a logical leaf, not a Doris STRUCT. Preserve its storage or
101+ // extension unchanged and let its SerDe validate the carrier shape and encoded payload.
102+ if (type->get_primitive_type () == TYPE_VARIANT ) {
103+ return wire;
104+ }
67105 // IPv4 is a 32-bit bit pattern, including addresses above INT32_MAX. Numeric INT columns
68106 // still use the checked conversion below instead of inheriting this storage exception.
69107 if (type->get_primitive_type () == TYPE_IPV4 && wire->id () == arrow::Type::UINT32 ) {
@@ -113,17 +151,20 @@ arrow::Result<std::shared_ptr<arrow::DataType>> input_binding_type(
113151}
114152
115153arrow::Result<std::shared_ptr<arrow::Array>> mask_hidden_children (
116- std::shared_ptr<arrow::Array> array, arrow::compute::ExecContext* context);
154+ std::shared_ptr<arrow::Array> array, const std::shared_ptr<arrow::DataType>& binding,
155+ arrow::compute::ExecContext* context);
117156
118157arrow::Result<std::shared_ptr<arrow::Array>> mask_struct_children (
119- const std::shared_ptr<arrow::Array>& array, arrow::compute::ExecContext* context) {
158+ const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& binding,
159+ arrow::compute::ExecContext* context) {
120160 const auto & structure = static_cast <const arrow::StructArray&>(*array);
121161 auto data = array->data ()->Copy ();
122162 for (int i = 0 ; i < structure.num_fields (); ++i) {
123163 // A NULL struct can have valid child payloads. Combine ancestor validity before
124164 // checked casts can inspect those logically hidden values.
125165 ARROW_ASSIGN_OR_RAISE (auto child, structure.GetFlattenedField (i, context->memory_pool ()));
126- ARROW_ASSIGN_OR_RAISE (child, mask_hidden_children (std::move (child), context));
166+ ARROW_ASSIGN_OR_RAISE (
167+ child, mask_hidden_children (std::move (child), binding->field (i)->type (), context));
127168 data->child_data [i] = child->data ();
128169 }
129170 return arrow::MakeArray (data);
@@ -150,25 +191,41 @@ arrow::Result<std::shared_ptr<arrow::Array>> compact_null_containers(
150191}
151192
152193arrow::Result<std::shared_ptr<arrow::Array>> mask_list_children (
153- std::shared_ptr<arrow::Array> array, arrow::compute::ExecContext* context) {
194+ std::shared_ptr<arrow::Array> array, const std::shared_ptr<arrow::DataType>& binding,
195+ arrow::compute::ExecContext* context) {
154196 ARROW_ASSIGN_OR_RAISE (array, compact_null_containers (std::move (array), context));
155197 auto data = array->data ()->Copy ();
156- ARROW_ASSIGN_OR_RAISE (auto child,
157- mask_hidden_children ( arrow::MakeArray (data-> child_data [ 0 ] ), context));
198+ ARROW_ASSIGN_OR_RAISE (auto child, mask_hidden_children ( arrow::MakeArray (data-> child_data [ 0 ]),
199+ binding-> field ( 0 )-> type ( ), context));
158200 data->child_data [0 ] = child->data ();
159201 return arrow::MakeArray (data);
160202}
161203
162204arrow::Result<std::shared_ptr<arrow::Array>> mask_hidden_children (
163- std::shared_ptr<arrow::Array> array, arrow::compute::ExecContext* context) {
205+ std::shared_ptr<arrow::Array> array, const std::shared_ptr<arrow::DataType>& binding,
206+ arrow::compute::ExecContext* context) {
207+ // Only conversions need hidden payloads masked. Descending into an unchanged VARIANT
208+ // carrier would make its non-nullable metadata/value fields NULL under a SQL NULL parent,
209+ // and Arrow's otherwise identical struct view would reject those synthetic child NULLs.
210+ if (array->type ()->Equals (binding)) {
211+ return array;
212+ }
213+ const bool converts_list_offsets = binding->id () == arrow::Type::LIST &&
214+ (array->type_id () == arrow::Type::LARGE_LIST ||
215+ array->type_id () == arrow::Type::FIXED_SIZE_LIST );
216+ // A STRUCT sent for a STRING slot has no corresponding target children. Leave mismatched
217+ // container kinds to Arrow's cast error rather than indexing nonexistent binding fields.
218+ if (array->type_id () != binding->id () && !converts_list_offsets) {
219+ return array;
220+ }
164221 switch (array->type_id ()) {
165222 case arrow::Type::STRUCT :
166- return mask_struct_children (array, context);
223+ return mask_struct_children (array, binding, context);
167224 case arrow::Type::LIST :
168225 case arrow::Type::LARGE_LIST :
169226 case arrow::Type::FIXED_SIZE_LIST :
170227 case arrow::Type::MAP :
171- return mask_list_children (std::move (array), context);
228+ return mask_list_children (std::move (array), binding, context);
172229 default :
173230 return array;
174231 }
@@ -275,7 +332,7 @@ Status ArrowStreamReader::init(RuntimeState* state) {
275332 }
276333 for (const auto * slot : _file_slots) {
277334 std::shared_ptr<arrow::DataType> type;
278- RETURN_IF_ERROR (DorisArrowSchemaConvertor (state->timezone ())
335+ RETURN_IF_ERROR (ArrowLoadSchemaConvertor (state->timezone ())
279336 .convert_to_arrow_type (slot->type (), &type));
280337 _arrow_types.push_back (std::move (type));
281338 }
@@ -352,8 +409,8 @@ Status ArrowStreamReader::_prepare_arrays() {
352409 // Bind differing numeric widths before concrete SerDe array casts. Timestamp units
353410 // stay on the wire even inside containers, so Doris retains its own rounding semantics.
354411 if (!array->type ()->Equals (binding)) {
355- RETURN_DORIS_STATUS_IF_RESULT_ERROR (array,
356- mask_hidden_children (std::move (array), &context));
412+ RETURN_DORIS_STATUS_IF_RESULT_ERROR (
413+ array, mask_hidden_children (std::move (array), binding , &context));
357414 auto options = arrow::compute::CastOptions::Safe ();
358415 // The Arrow load protocol permits binary payloads in Doris STRING columns.
359416 options.allow_invalid_utf8 = true ;
0 commit comments