Skip to content

Commit 4f90a5a

Browse files
committed
[fix](flight) Preserve native Variant document types and validate edge values
1 parent ae5067f commit 4f90a5a

5 files changed

Lines changed: 341 additions & 40 deletions

File tree

‎be/src/core/data_type_serde/data_type_variant_serde.cpp‎

Lines changed: 125 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,12 @@
1919

2020
#include <arrow/array/builder_binary.h>
2121

22+
#include <algorithm>
2223
#include <cmath>
2324
#include <cstdint>
24-
#include <limits>
2525
#include <string>
26+
#include <string_view>
27+
#include <vector>
2628

2729
#include "common/cast_set.h"
2830
#include "common/config.h"
@@ -55,6 +57,10 @@
5557
namespace doris {
5658
namespace {
5759

60+
Status append_legacy_arrow_document(const ColumnVariant& column, size_t index,
61+
VariantBatchBuilder::Row& output,
62+
const DataTypeSerDe::FormatOptions& options, size_t depth);
63+
5864
// Legacy CAST accepts more root families than V2 CAST. Encode their structure here so
5965
// Flight output does not reject valid roots or lose typed leaves through JSON reparsing.
6066
Status append_legacy_arrow_value(const IColumn& column, const DataTypePtr& type, size_t index,
@@ -89,11 +95,19 @@ Status append_legacy_arrow_value(const IColumn& column, const DataTypePtr& type,
8995
} else if (primitive == TYPE_TIMEV2) {
9096
// TIMEV2 already stores microseconds; treating its physical double as a number loses its type.
9197
const double micros = assert_cast<const ColumnTimeV2&>(column).get_data()[index];
92-
if (!std::isfinite(micros) ||
93-
std::abs(micros) >= static_cast<double>(std::numeric_limits<int64_t>::max())) {
94-
return Status::InvalidArgument("Invalid native Arrow Variant TIMEV2 value");
98+
// Parquet TIME is a time of day, whereas Doris TIME also represents signed durations.
99+
// Reject unrepresentable durations instead of wrapping them or emitting invalid TIME values.
100+
constexpr int64_t micros_per_day = 86400000000;
101+
if (!std::isfinite(micros) || micros < 0 || micros >= micros_per_day ||
102+
std::llround(micros) >= micros_per_day) {
103+
return Status::NotSupported(
104+
"Native Arrow Variant TIMEV2 requires a time in [00:00:00, 24:00:00); "
105+
"use enable_arrow_flight_sql_native_variant=false for UTF8 output");
95106
}
96107
output.add_time_ntz_micros(std::llround(micros));
108+
} else if (primitive == TYPE_VARBINARY) {
109+
// Binary leaves must retain arbitrary bytes, including NUL and non-UTF8 data.
110+
output.add_binary(column.get_data_at(index));
97111
} else if (primitive == TYPE_JSONB) {
98112
jsonb_to_variant(column.get_data_at(index), output);
99113
} else if (primitive == TYPE_ARRAY) {
@@ -113,11 +127,13 @@ Status append_legacy_arrow_value(const IColumn& column, const DataTypePtr& type,
113127
auto scope = output.start_object();
114128
for (size_t element = map.get_offsets()[static_cast<ssize_t>(index) - 1];
115129
element < map.get_offsets()[index]; ++element) {
116-
// Variant objects have textual keys, matching the legacy document representation.
117-
auto key =
118-
map.get_keys().is_null_at(element)
119-
? std::string("null")
120-
: map_type.get_key_type()->to_string(map.get_keys(), element, options);
130+
// Variant object keys cannot distinguish SQL NULL from the literal string "null".
131+
if (map.get_keys().is_null_at(element)) {
132+
return Status::NotSupported(
133+
"Native Arrow Variant cannot represent MAP with NULL keys; "
134+
"use enable_arrow_flight_sql_native_variant=false for UTF8 output");
135+
}
136+
auto key = map_type.get_key_type()->to_string(map.get_keys(), element, options);
121137
scope.add_key({key.data(), key.size()});
122138
RETURN_IF_ERROR(append_legacy_arrow_value(map.get_values(), map_type.get_value_type(),
123139
element, output, options, depth + 1));
@@ -144,15 +160,7 @@ Status append_legacy_arrow_value(const IColumn& column, const DataTypePtr& type,
144160
return append_legacy_arrow_value(*legacy->get_root(), legacy->get_root_type(),
145161
index, output, options, depth);
146162
}
147-
std::string json;
148-
legacy->serialize_one_row_to_string(index, &json, options);
149-
JsonToVariantOptions parse_options;
150-
parse_options.throw_on_invalid_json = true;
151-
parse_options.max_json_key_length = std::numeric_limits<uint32_t>::max();
152-
JsonStringToVariantEncoder encoder(parse_options);
153-
RETURN_IF_ERROR(encoder.try_add_json({json.data(), json.size()}));
154-
auto encoded = encoder.finish_batch();
155-
output.add_value(encoded.value_at(0));
163+
RETURN_IF_ERROR(append_legacy_arrow_document(*legacy, index, output, options, depth));
156164
} else {
157165
visit_variant_v2_values(
158166
column, index, index + 1, {}, [&](size_t) { output.add_null(); },
@@ -165,6 +173,94 @@ Status append_legacy_arrow_value(const IColumn& column, const DataTypePtr& type,
165173
return Status::OK();
166174
}
167175

176+
// Assemble the same flattened document paths as legacy JSON output, but encode each
177+
// leaf with its stored type. JSON reparsing loses decimal precision and date identity.
178+
Status append_legacy_arrow_document(const ColumnVariant& column, size_t index,
179+
VariantBatchBuilder::Row& output,
180+
const DataTypeSerDe::FormatOptions& options, size_t depth) {
181+
struct Field {
182+
std::string path;
183+
ColumnPtr values;
184+
DataTypePtr type;
185+
size_t row;
186+
};
187+
std::vector<Field> fields;
188+
auto append_serialized = [&](const ColumnMap& map) {
189+
const auto& keys = assert_cast<const ColumnString&>(map.get_keys());
190+
const auto& values = assert_cast<const ColumnString&>(map.get_values());
191+
for (size_t i = map.get_offsets()[static_cast<ssize_t>(index) - 1];
192+
i < map.get_offsets()[index]; ++i) {
193+
ColumnVariant::Subcolumn subcolumn(0, true);
194+
subcolumn.deserialize_from_binary_column(&values, i);
195+
subcolumn.finalize();
196+
fields.push_back({keys.get_data_at(i).to_string(), subcolumn.get_finalized_column_ptr(),
197+
subcolumn.get_least_common_type(), 0});
198+
}
199+
};
200+
const auto& snapshot = assert_cast<const ColumnMap&>(*column.get_doc_value_column());
201+
if (snapshot.get_offsets()[index] != 0) {
202+
// Document snapshots are authoritative, including empty rows after a populated snapshot.
203+
append_serialized(snapshot);
204+
} else {
205+
for (const auto& subcolumn : column.get_subcolumns()) {
206+
if (subcolumn->path.empty() || subcolumn->data.is_null_at(index) ||
207+
subcolumn->data.is_empty_nested(index)) {
208+
continue;
209+
}
210+
fields.push_back({subcolumn->path.get_path(),
211+
subcolumn->data.get_finalized_column_ptr(),
212+
subcolumn->data.get_least_common_type(), index});
213+
}
214+
append_serialized(assert_cast<const ColumnMap&>(*column.get_sparse_column()));
215+
}
216+
std::sort(fields.begin(), fields.end(),
217+
[](const auto& a, const auto& b) { return a.path < b.path; });
218+
std::vector<std::string_view> prefix;
219+
std::vector<VariantBatchBuilder::Row::ObjectScope> objects;
220+
objects.push_back(output.start_object());
221+
for (const auto& field : fields) {
222+
std::vector<std::string_view> parts;
223+
std::string_view path(field.path);
224+
while (true) {
225+
const auto dot = path.find('.');
226+
parts.push_back(path.substr(0, dot));
227+
if (depth + parts.size() > VARIANT_MAX_NESTING_DEPTH) {
228+
return Status::NotSupported(
229+
"Native Arrow Variant nesting exceeds {}; "
230+
"use enable_arrow_flight_sql_native_variant=false for UTF8 output",
231+
VARIANT_MAX_NESTING_DEPTH);
232+
}
233+
if (dot == std::string_view::npos) {
234+
break;
235+
}
236+
path.remove_prefix(dot + 1);
237+
}
238+
size_t common = 0;
239+
while (common < prefix.size() && common + 1 < parts.size() &&
240+
prefix[common] == parts[common]) {
241+
++common;
242+
}
243+
while (prefix.size() > common) {
244+
objects.back().finish();
245+
objects.pop_back();
246+
prefix.pop_back();
247+
}
248+
for (size_t i = common; i + 1 < parts.size(); ++i) {
249+
objects.back().add_key({parts[i].data(), parts[i].size()});
250+
objects.push_back(output.start_object());
251+
prefix.push_back(parts[i]);
252+
}
253+
objects.back().add_key({parts.back().data(), parts.back().size()});
254+
RETURN_IF_ERROR(append_legacy_arrow_value(*field.values, field.type, field.row, output,
255+
options, depth + parts.size()));
256+
}
257+
while (!objects.empty()) {
258+
objects.back().finish();
259+
objects.pop_back();
260+
}
261+
return Status::OK();
262+
}
263+
168264
template <typename BuilderType>
169265
Status write_variant_column_to_arrow_impl(const IColumn& column, const ColumnVariant& var,
170266
const NullMap* null_map, BuilderType& builder,
@@ -282,7 +378,7 @@ Status DataTypeVariantSerDe::write_column_to_arrow(const IColumn& column, const
282378
const cctz::time_zone& ctz) const {
283379
const auto* var = check_and_get_column<ColumnVariant>(column);
284380
if (array_builder->type()->id() == arrow::Type::STRUCT) {
285-
// Legacy documents need JSON conversion; typed scalar roots can keep their type.
381+
// Keep legacy scalar and document leaves in their original types.
286382
// The outer null map must remain SQL NULL on the wire.
287383
if (start < 0 || end < start || end > column.size() ||
288384
(null_map != nullptr && end > null_map->size())) {
@@ -342,28 +438,20 @@ Status DataTypeVariantSerDe::write_column_to_arrow(const IColumn& column, const
342438
}
343439
ColumnPtr documents;
344440
if (has_documents || !has_roots) {
345-
JsonToVariantOptions parse_options;
346-
parse_options.throw_on_invalid_json = true;
347-
// Stored keys were already accepted at ingestion; mutable parse limits must not reject reads.
348-
parse_options.max_json_key_length = std::numeric_limits<uint32_t>::max();
349-
JsonStringToVariantEncoder encoder(parse_options);
441+
VariantBatchBuilder builder(VariantBatchBuilder::ReserveHint {.rows = rows});
350442
FormatOptions options;
351443
options.timezone = &ctz;
352-
for (size_t row = 0; row < rows; ++row) {
353-
std::string json = "null";
354-
if (root_mask[row] && !selected_nulls[row]) {
355-
var->serialize_one_row_to_string(start + row, &json, options);
356-
}
357-
auto status = encoder.try_add_json({json.data(), json.size()});
358-
if (!status.ok()) {
359-
return Status::NotSupported(
360-
"Cannot encode legacy document as native Arrow Variant: {}. "
361-
"Native documents are limited to {} nested levels; "
362-
"use enable_arrow_flight_sql_native_variant=false for UTF8 output",
363-
status.to_string(), VARIANT_MAX_NESTING_DEPTH);
444+
for (size_t index = 0; index < rows; ++index) {
445+
auto row = builder.begin_row();
446+
if (root_mask[index] && !selected_nulls[index]) {
447+
RETURN_IF_ERROR(
448+
append_legacy_arrow_document(*var, start + index, row, options, 0));
449+
} else {
450+
row.add_null();
364451
}
452+
row.finish();
365453
}
366-
auto values = encoder.finish_batch();
454+
auto values = builder.finish_batch();
367455
auto encoded = ColumnVariantV2::create();
368456
encoded->insert_encoded_batch(values);
369457
documents = std::move(encoded);

0 commit comments

Comments
 (0)