Skip to content

Commit d7bd729

Browse files
committed
[fix](load) Preserve native Arrow states and empty-file diagnostics
### What problem does this PR solve? Native VARIANT carriers were rebound to UTF-8 and serialized fixed-length AGG_STATE values lacked an Arrow input mapping. Add a dedicated input schema converter, retain native carriers recursively, and derive fixed-state widths from the actual serialization column. Mask only child subtrees requiring conversion and return explicit errors for incompatible input containers. Restore existing empty-file schema RPC diagnostics for plain and compressed CSV inputs, including header-name versus inferred-column-number errors. ### Release note Preserve native VARIANT and aggregate states during Arrow Stream Load and retain existing empty-file schema discovery errors. ### Check List (For Author) - Test: 212 ASAN BE unit tests, six regression suites, BE production build, clang-format 16, changed-line clang-tidy, and build hygiene checks. - Behavior changed: Native Arrow values load using their input storage contracts; empty compressed files retain existing schema errors. - Does this need documentation? No; existing load interfaces are retained.
1 parent 4df2737 commit d7bd729

4 files changed

Lines changed: 335 additions & 20 deletions

File tree

‎be/src/format_v2/arrow/arrow_stream_reader.cpp‎

Lines changed: 70 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@
2929
#include <utility>
3030

3131
#include "core/block/block.h"
32+
#include "core/column/column_fixed_length_object.h"
33+
#include "core/data_type/data_type_agg_state.h"
3234
#include "core/data_type/data_type_array.h"
3335
#include "core/data_type/data_type_map.h"
3436
#include "core/data_type/data_type_nullable.h"
@@ -48,6 +50,37 @@ namespace doris::format::arrow_stream {
4850

4951
namespace {
5052

53+
class ArrowLoadSchemaConvertor final : public DorisArrowSchemaConvertor {
54+
public:
55+
using DorisArrowSchemaConvertor::DorisArrowSchemaConvertor;
56+
57+
Status convert_to_arrow_type(const DataTypePtr& origin_type,
58+
std::shared_ptr<arrow::DataType>* result) const override {
59+
const auto type = remove_nullable(origin_type);
60+
// VARIANT output may be UTF-8, but its input SerDe requires the native binary carrier,
61+
// including when VARIANT is nested in an ARRAY, MAP, or STRUCT.
62+
if (type->get_primitive_type() == TYPE_VARIANT) {
63+
*result = arrow::struct_({arrow::field("metadata", arrow::binary(), false),
64+
arrow::field("value", arrow::binary(), false)});
65+
return Status::OK();
66+
}
67+
if (type->get_primitive_type() == TYPE_AGG_STATE) {
68+
const auto& state = assert_cast<const DataTypeAggState&>(*type);
69+
const auto serialized_type = remove_nullable(state.get_serialized_type());
70+
// FixedLengthObject's type omits its width. The serialization column owns that
71+
// contract (e.g. count(INT NOT NULL) uses eight bytes), not the runtime state size.
72+
if (serialized_type->get_primitive_type() == TYPE_FIXED_LENGTH_OBJECT) {
73+
const auto column = state.create_column();
74+
const auto width = assert_cast<const ColumnFixedLengthObject&>(*column).item_size();
75+
*result = arrow::fixed_size_binary(cast_set<int32_t>(width));
76+
return Status::OK();
77+
}
78+
return convert_to_arrow_type(serialized_type, result);
79+
}
80+
return DorisArrowSchemaConvertor::convert_to_arrow_type(type, result);
81+
}
82+
};
83+
5184
arrow::Result<std::shared_ptr<arrow::DataType>> input_binding_type(
5285
const std::shared_ptr<arrow::DataType>& wire,
5386
const std::shared_ptr<arrow::DataType>& declared, const DataTypePtr& doris_type) {
@@ -63,7 +96,12 @@ arrow::Result<std::shared_ptr<arrow::DataType>> input_binding_type(
6396
if (wire->id() == arrow::Type::DATE64 && declared->id() == arrow::Type::DATE32) {
6497
return wire;
6598
}
66-
const auto type = remove_nullable(doris_type);
99+
const auto type = remove_nullable(get_serialized_type(remove_nullable(doris_type)));
100+
// A native VARIANT carrier is a logical leaf, not a Doris STRUCT. Preserve its storage or
101+
// extension unchanged and let its SerDe validate the carrier shape and encoded payload.
102+
if (type->get_primitive_type() == TYPE_VARIANT) {
103+
return wire;
104+
}
67105
// IPv4 is a 32-bit bit pattern, including addresses above INT32_MAX. Numeric INT columns
68106
// still use the checked conversion below instead of inheriting this storage exception.
69107
if (type->get_primitive_type() == TYPE_IPV4 && wire->id() == arrow::Type::UINT32) {
@@ -113,17 +151,20 @@ arrow::Result<std::shared_ptr<arrow::DataType>> input_binding_type(
113151
}
114152

115153
arrow::Result<std::shared_ptr<arrow::Array>> mask_hidden_children(
116-
std::shared_ptr<arrow::Array> array, arrow::compute::ExecContext* context);
154+
std::shared_ptr<arrow::Array> array, const std::shared_ptr<arrow::DataType>& binding,
155+
arrow::compute::ExecContext* context);
117156

118157
arrow::Result<std::shared_ptr<arrow::Array>> mask_struct_children(
119-
const std::shared_ptr<arrow::Array>& array, arrow::compute::ExecContext* context) {
158+
const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& binding,
159+
arrow::compute::ExecContext* context) {
120160
const auto& structure = static_cast<const arrow::StructArray&>(*array);
121161
auto data = array->data()->Copy();
122162
for (int i = 0; i < structure.num_fields(); ++i) {
123163
// A NULL struct can have valid child payloads. Combine ancestor validity before
124164
// checked casts can inspect those logically hidden values.
125165
ARROW_ASSIGN_OR_RAISE(auto child, structure.GetFlattenedField(i, context->memory_pool()));
126-
ARROW_ASSIGN_OR_RAISE(child, mask_hidden_children(std::move(child), context));
166+
ARROW_ASSIGN_OR_RAISE(
167+
child, mask_hidden_children(std::move(child), binding->field(i)->type(), context));
127168
data->child_data[i] = child->data();
128169
}
129170
return arrow::MakeArray(data);
@@ -150,25 +191,41 @@ arrow::Result<std::shared_ptr<arrow::Array>> compact_null_containers(
150191
}
151192

152193
arrow::Result<std::shared_ptr<arrow::Array>> mask_list_children(
153-
std::shared_ptr<arrow::Array> array, arrow::compute::ExecContext* context) {
194+
std::shared_ptr<arrow::Array> array, const std::shared_ptr<arrow::DataType>& binding,
195+
arrow::compute::ExecContext* context) {
154196
ARROW_ASSIGN_OR_RAISE(array, compact_null_containers(std::move(array), context));
155197
auto data = array->data()->Copy();
156-
ARROW_ASSIGN_OR_RAISE(auto child,
157-
mask_hidden_children(arrow::MakeArray(data->child_data[0]), context));
198+
ARROW_ASSIGN_OR_RAISE(auto child, mask_hidden_children(arrow::MakeArray(data->child_data[0]),
199+
binding->field(0)->type(), context));
158200
data->child_data[0] = child->data();
159201
return arrow::MakeArray(data);
160202
}
161203

162204
arrow::Result<std::shared_ptr<arrow::Array>> mask_hidden_children(
163-
std::shared_ptr<arrow::Array> array, arrow::compute::ExecContext* context) {
205+
std::shared_ptr<arrow::Array> array, const std::shared_ptr<arrow::DataType>& binding,
206+
arrow::compute::ExecContext* context) {
207+
// Only conversions need hidden payloads masked. Descending into an unchanged VARIANT
208+
// carrier would make its non-nullable metadata/value fields NULL under a SQL NULL parent,
209+
// and Arrow's otherwise identical struct view would reject those synthetic child NULLs.
210+
if (array->type()->Equals(binding)) {
211+
return array;
212+
}
213+
const bool converts_list_offsets = binding->id() == arrow::Type::LIST &&
214+
(array->type_id() == arrow::Type::LARGE_LIST ||
215+
array->type_id() == arrow::Type::FIXED_SIZE_LIST);
216+
// A STRUCT sent for a STRING slot has no corresponding target children. Leave mismatched
217+
// container kinds to Arrow's cast error rather than indexing nonexistent binding fields.
218+
if (array->type_id() != binding->id() && !converts_list_offsets) {
219+
return array;
220+
}
164221
switch (array->type_id()) {
165222
case arrow::Type::STRUCT:
166-
return mask_struct_children(array, context);
223+
return mask_struct_children(array, binding, context);
167224
case arrow::Type::LIST:
168225
case arrow::Type::LARGE_LIST:
169226
case arrow::Type::FIXED_SIZE_LIST:
170227
case arrow::Type::MAP:
171-
return mask_list_children(std::move(array), context);
228+
return mask_list_children(std::move(array), binding, context);
172229
default:
173230
return array;
174231
}
@@ -275,7 +332,7 @@ Status ArrowStreamReader::init(RuntimeState* state) {
275332
}
276333
for (const auto* slot : _file_slots) {
277334
std::shared_ptr<arrow::DataType> type;
278-
RETURN_IF_ERROR(DorisArrowSchemaConvertor(state->timezone())
335+
RETURN_IF_ERROR(ArrowLoadSchemaConvertor(state->timezone())
279336
.convert_to_arrow_type(slot->type(), &type));
280337
_arrow_types.push_back(std::move(type));
281338
}
@@ -352,8 +409,8 @@ Status ArrowStreamReader::_prepare_arrays() {
352409
// Bind differing numeric widths before concrete SerDe array casts. Timestamp units
353410
// stay on the wire even inside containers, so Doris retains its own rounding semantics.
354411
if (!array->type()->Equals(binding)) {
355-
RETURN_DORIS_STATUS_IF_RESULT_ERROR(array,
356-
mask_hidden_children(std::move(array), &context));
412+
RETURN_DORIS_STATUS_IF_RESULT_ERROR(
413+
array, mask_hidden_children(std::move(array), binding, &context));
357414
auto options = arrow::compute::CastOptions::Safe();
358415
// The Arrow load protocol permits binary payloads in Doris STRING columns.
359416
options.allow_invalid_utf8 = true;

‎be/src/format_v2/delimited_text/delimited_text_reader.cpp‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -301,18 +301,21 @@ Status DelimitedTextReader::infer_schema(RuntimeState* state,
301301
Slice line;
302302
bool eof = false;
303303
RETURN_IF_ERROR(_read_next_line(&line, &eof));
304+
const auto header = to_lower(_scan_params->file_attributes.header_type);
305+
const bool has_names =
306+
header == BeConsts::CSV_WITH_NAMES || header == BeConsts::CSV_WITH_NAMES_AND_TYPES;
304307
if (eof || line.empty()) {
305-
return Status::InternalError("The first line is empty, cannot infer column names");
308+
// Empty compressed files reach this same check after decompression. Preserve the
309+
// schema RPC's existing diagnostics for callers such as the empty-LZOP TVF case.
310+
return Status::InternalError("The first line is empty, can not parse column {}",
311+
has_names ? "names" : "numbers");
306312
}
307313
// TEXT scans allow raw bytes, but schema RPCs must retain their UTF-8 validation contract
308314
// (for example, a header containing 0xFF must fail unless validation is explicitly disabled).
309315
if (!validate_utf8(*_scan_params, line.data, line.size)) {
310316
return Status::InternalError("Only support csv data in utf8 codec");
311317
}
312318
_split_line(line);
313-
const auto header = to_lower(_scan_params->file_attributes.header_type);
314-
const bool has_names =
315-
header == BeConsts::CSV_WITH_NAMES || header == BeConsts::CSV_WITH_NAMES_AND_TYPES;
316319
schema->clear();
317320
for (size_t i = 0; i < _split_values.size(); ++i) {
318321
const auto name = has_names ? _split_values[i].to_string() : fmt::format("c{}", i + 1);

‎be/test/format_v2/schema_reader_test.cpp‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,10 +141,32 @@ TEST_F(SchemaInferenceV2Test, JsonRejectsMalformedInputAndInvalidShape) {
141141

142142
TEST_F(SchemaInferenceV2Test, CsvRejectsEmptyFirstLineAndNonzeroOffset) {
143143
EXPECT_FALSE(infer("").ok());
144-
EXPECT_FALSE(infer("\n1,value\n").ok());
144+
const auto status = infer("\n1,value\n");
145+
EXPECT_FALSE(status.ok());
146+
EXPECT_NE(status.to_string().find("The first line is empty, can not parse column numbers"),
147+
std::string::npos)
148+
<< status;
145149
EXPECT_FALSE(infer("1,value\n", 1).ok());
146150
}
147151

152+
TEST_F(SchemaInferenceV2Test, EmptyLzopRetainsTheSchemaRpcError) {
153+
// A valid LZOP header (no filename, zero timestamps), Adler-32, and an end-of-stream block.
154+
const uint8_t stream[] = {0x89, 0x4c, 0x5a, 0x4f, 0x00, 0x0d, 0x0a, 0x1a, 0x0a, 0x10, 0x30,
155+
0x20, 0xa0, 0x09, 0x40, 0x01, 0x05, 0x00, 0x00, 0x00, 0x01, 0x00,
156+
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
157+
0x00, 0x1d, 0x01, 0x01, 0x51, 0x00, 0x00, 0x00, 0x00};
158+
params.__set_format_type(TFileFormatType::FORMAT_CSV_LZOP);
159+
for (const auto* header : {"", "csv_with_names", "csv_with_names_and_types"}) {
160+
attributes.__set_header_type(header);
161+
const auto status =
162+
infer(std::string(reinterpret_cast<const char*>(stream), sizeof(stream)));
163+
EXPECT_FALSE(status.ok());
164+
const auto expected = std::string("The first line is empty, can not parse column ") +
165+
(*header == '\0' ? "numbers" : "names");
166+
EXPECT_NE(status.to_string().find(expected), std::string::npos) << status;
167+
}
168+
}
169+
148170
TEST(SchemaReaderV2Test, InfersTextAndJsonWithoutPlannedSlots) {
149171
test::ScopedTempDirectory temp("schema_v2");
150172
struct Case {

0 commit comments

Comments
 (0)