1919
2020#include < arrow/array/builder_binary.h>
2121
22+ #include < algorithm>
2223#include < cmath>
2324#include < cstdint>
24- #include < limits>
2525#include < string>
26+ #include < string_view>
27+ #include < vector>
2628
2729#include " common/cast_set.h"
2830#include " common/config.h"
5557namespace doris {
5658namespace {
5759
60+ Status append_legacy_arrow_document (const ColumnVariant& column, size_t index,
61+ VariantBatchBuilder::Row& output,
62+ const DataTypeSerDe::FormatOptions& options, size_t depth);
63+
5864// Legacy CAST accepts more root families than V2 CAST. Encode their structure here so
5965// Flight output does not reject valid roots or lose typed leaves through JSON reparsing.
6066Status append_legacy_arrow_value (const IColumn& column, const DataTypePtr& type, size_t index,
@@ -89,11 +95,19 @@ Status append_legacy_arrow_value(const IColumn& column, const DataTypePtr& type,
8995 } else if (primitive == TYPE_TIMEV2 ) {
9096 // TIMEV2 already stores microseconds; treating its physical double as a number loses its type.
9197 const double micros = assert_cast<const ColumnTimeV2&>(column).get_data ()[index];
92- if (!std::isfinite (micros) ||
93- std::abs (micros) >= static_cast <double >(std::numeric_limits<int64_t >::max ())) {
94- return Status::InvalidArgument (" Invalid native Arrow Variant TIMEV2 value" );
98+ // Parquet TIME is a time of day, whereas Doris TIME also represents signed durations.
99+ // Reject unrepresentable durations instead of wrapping them or emitting invalid TIME values.
100+ constexpr int64_t micros_per_day = 86400000000 ;
101+ if (!std::isfinite (micros) || micros < 0 || micros >= micros_per_day ||
102+ std::llround (micros) >= micros_per_day) {
103+ return Status::NotSupported (
104+ " Native Arrow Variant TIMEV2 requires a time in [00:00:00, 24:00:00); "
105+ " use enable_arrow_flight_sql_native_variant=false for UTF8 output" );
95106 }
96107 output.add_time_ntz_micros (std::llround (micros));
108+ } else if (primitive == TYPE_VARBINARY ) {
109+ // Binary leaves must retain arbitrary bytes, including NUL and non-UTF8 data.
110+ output.add_binary (column.get_data_at (index));
97111 } else if (primitive == TYPE_JSONB ) {
98112 jsonb_to_variant (column.get_data_at (index), output);
99113 } else if (primitive == TYPE_ARRAY ) {
@@ -113,11 +127,13 @@ Status append_legacy_arrow_value(const IColumn& column, const DataTypePtr& type,
113127 auto scope = output.start_object ();
114128 for (size_t element = map.get_offsets ()[static_cast <ssize_t >(index) - 1 ];
115129 element < map.get_offsets ()[index]; ++element) {
116- // Variant objects have textual keys, matching the legacy document representation.
117- auto key =
118- map.get_keys ().is_null_at (element)
119- ? std::string (" null" )
120- : map_type.get_key_type ()->to_string (map.get_keys (), element, options);
130+ // Variant object keys cannot distinguish SQL NULL from the literal string "null".
131+ if (map.get_keys ().is_null_at (element)) {
132+ return Status::NotSupported (
133+ " Native Arrow Variant cannot represent MAP with NULL keys; "
134+ " use enable_arrow_flight_sql_native_variant=false for UTF8 output" );
135+ }
136+ auto key = map_type.get_key_type ()->to_string (map.get_keys (), element, options);
121137 scope.add_key ({key.data (), key.size ()});
122138 RETURN_IF_ERROR (append_legacy_arrow_value (map.get_values (), map_type.get_value_type (),
123139 element, output, options, depth + 1 ));
@@ -144,15 +160,7 @@ Status append_legacy_arrow_value(const IColumn& column, const DataTypePtr& type,
144160 return append_legacy_arrow_value (*legacy->get_root (), legacy->get_root_type (),
145161 index, output, options, depth);
146162 }
147- std::string json;
148- legacy->serialize_one_row_to_string (index, &json, options);
149- JsonToVariantOptions parse_options;
150- parse_options.throw_on_invalid_json = true ;
151- parse_options.max_json_key_length = std::numeric_limits<uint32_t >::max ();
152- JsonStringToVariantEncoder encoder (parse_options);
153- RETURN_IF_ERROR (encoder.try_add_json ({json.data (), json.size ()}));
154- auto encoded = encoder.finish_batch ();
155- output.add_value (encoded.value_at (0 ));
163+ RETURN_IF_ERROR (append_legacy_arrow_document (*legacy, index, output, options, depth));
156164 } else {
157165 visit_variant_v2_values (
158166 column, index, index + 1 , {}, [&](size_t ) { output.add_null (); },
@@ -165,6 +173,94 @@ Status append_legacy_arrow_value(const IColumn& column, const DataTypePtr& type,
165173 return Status::OK ();
166174}
167175
176+ // Assemble the same flattened document paths as legacy JSON output, but encode each
177+ // leaf with its stored type. JSON reparsing loses decimal precision and date identity.
178+ Status append_legacy_arrow_document (const ColumnVariant& column, size_t index,
179+ VariantBatchBuilder::Row& output,
180+ const DataTypeSerDe::FormatOptions& options, size_t depth) {
181+ struct Field {
182+ std::string path;
183+ ColumnPtr values;
184+ DataTypePtr type;
185+ size_t row;
186+ };
187+ std::vector<Field> fields;
188+ auto append_serialized = [&](const ColumnMap& map) {
189+ const auto & keys = assert_cast<const ColumnString&>(map.get_keys ());
190+ const auto & values = assert_cast<const ColumnString&>(map.get_values ());
191+ for (size_t i = map.get_offsets ()[static_cast <ssize_t >(index) - 1 ];
192+ i < map.get_offsets ()[index]; ++i) {
193+ ColumnVariant::Subcolumn subcolumn (0 , true );
194+ subcolumn.deserialize_from_binary_column (&values, i);
195+ subcolumn.finalize ();
196+ fields.push_back ({keys.get_data_at (i).to_string (), subcolumn.get_finalized_column_ptr (),
197+ subcolumn.get_least_common_type (), 0 });
198+ }
199+ };
200+ const auto & snapshot = assert_cast<const ColumnMap&>(*column.get_doc_value_column ());
201+ if (snapshot.get_offsets ()[index] != 0 ) {
202+ // Document snapshots are authoritative, including empty rows after a populated snapshot.
203+ append_serialized (snapshot);
204+ } else {
205+ for (const auto & subcolumn : column.get_subcolumns ()) {
206+ if (subcolumn->path .empty () || subcolumn->data .is_null_at (index) ||
207+ subcolumn->data .is_empty_nested (index)) {
208+ continue ;
209+ }
210+ fields.push_back ({subcolumn->path .get_path (),
211+ subcolumn->data .get_finalized_column_ptr (),
212+ subcolumn->data .get_least_common_type (), index});
213+ }
214+ append_serialized (assert_cast<const ColumnMap&>(*column.get_sparse_column ()));
215+ }
216+ std::sort (fields.begin (), fields.end (),
217+ [](const auto & a, const auto & b) { return a.path < b.path ; });
218+ std::vector<std::string_view> prefix;
219+ std::vector<VariantBatchBuilder::Row::ObjectScope> objects;
220+ objects.push_back (output.start_object ());
221+ for (const auto & field : fields) {
222+ std::vector<std::string_view> parts;
223+ std::string_view path (field.path );
224+ while (true ) {
225+ const auto dot = path.find (' .' );
226+ parts.push_back (path.substr (0 , dot));
227+ if (depth + parts.size () > VARIANT_MAX_NESTING_DEPTH ) {
228+ return Status::NotSupported (
229+ " Native Arrow Variant nesting exceeds {}; "
230+ " use enable_arrow_flight_sql_native_variant=false for UTF8 output" ,
231+ VARIANT_MAX_NESTING_DEPTH );
232+ }
233+ if (dot == std::string_view::npos) {
234+ break ;
235+ }
236+ path.remove_prefix (dot + 1 );
237+ }
238+ size_t common = 0 ;
239+ while (common < prefix.size () && common + 1 < parts.size () &&
240+ prefix[common] == parts[common]) {
241+ ++common;
242+ }
243+ while (prefix.size () > common) {
244+ objects.back ().finish ();
245+ objects.pop_back ();
246+ prefix.pop_back ();
247+ }
248+ for (size_t i = common; i + 1 < parts.size (); ++i) {
249+ objects.back ().add_key ({parts[i].data (), parts[i].size ()});
250+ objects.push_back (output.start_object ());
251+ prefix.push_back (parts[i]);
252+ }
253+ objects.back ().add_key ({parts.back ().data (), parts.back ().size ()});
254+ RETURN_IF_ERROR (append_legacy_arrow_value (*field.values , field.type , field.row , output,
255+ options, depth + parts.size ()));
256+ }
257+ while (!objects.empty ()) {
258+ objects.back ().finish ();
259+ objects.pop_back ();
260+ }
261+ return Status::OK ();
262+ }
263+
168264template <typename BuilderType>
169265Status write_variant_column_to_arrow_impl (const IColumn& column, const ColumnVariant& var,
170266 const NullMap* null_map, BuilderType& builder,
@@ -282,7 +378,7 @@ Status DataTypeVariantSerDe::write_column_to_arrow(const IColumn& column, const
282378 const cctz::time_zone& ctz) const {
283379 const auto * var = check_and_get_column<ColumnVariant>(column);
284380 if (array_builder->type ()->id () == arrow::Type::STRUCT ) {
285- // Legacy documents need JSON conversion; typed scalar roots can keep their type .
381+ // Keep legacy scalar and document leaves in their original types .
286382 // The outer null map must remain SQL NULL on the wire.
287383 if (start < 0 || end < start || end > column.size () ||
288384 (null_map != nullptr && end > null_map->size ())) {
@@ -342,28 +438,20 @@ Status DataTypeVariantSerDe::write_column_to_arrow(const IColumn& column, const
342438 }
343439 ColumnPtr documents;
344440 if (has_documents || !has_roots) {
345- JsonToVariantOptions parse_options;
346- parse_options.throw_on_invalid_json = true ;
347- // Stored keys were already accepted at ingestion; mutable parse limits must not reject reads.
348- parse_options.max_json_key_length = std::numeric_limits<uint32_t >::max ();
349- JsonStringToVariantEncoder encoder (parse_options);
441+ VariantBatchBuilder builder (VariantBatchBuilder::ReserveHint {.rows = rows});
350442 FormatOptions options;
351443 options.timezone = &ctz;
352- for (size_t row = 0 ; row < rows; ++row) {
353- std::string json = " null" ;
354- if (root_mask[row] && !selected_nulls[row]) {
355- var->serialize_one_row_to_string (start + row, &json, options);
356- }
357- auto status = encoder.try_add_json ({json.data (), json.size ()});
358- if (!status.ok ()) {
359- return Status::NotSupported (
360- " Cannot encode legacy document as native Arrow Variant: {}. "
361- " Native documents are limited to {} nested levels; "
362- " use enable_arrow_flight_sql_native_variant=false for UTF8 output" ,
363- status.to_string (), VARIANT_MAX_NESTING_DEPTH );
444+ for (size_t index = 0 ; index < rows; ++index) {
445+ auto row = builder.begin_row ();
446+ if (root_mask[index] && !selected_nulls[index]) {
447+ RETURN_IF_ERROR (
448+ append_legacy_arrow_document (*var, start + index, row, options, 0 ));
449+ } else {
450+ row.add_null ();
364451 }
452+ row.finish ();
365453 }
366- auto values = encoder .finish_batch ();
454+ auto values = builder .finish_batch ();
367455 auto encoded = ColumnVariantV2::create ();
368456 encoded->insert_encoded_batch (values);
369457 documents = std::move (encoded);
0 commit comments