Skip to content

Commit c64d2d7

Browse files
committed
refactor some be code
1 parent aeb7e1a commit c64d2d7

10 files changed

Lines changed: 607 additions & 468 deletions

‎be/src/format_v2/lance/lance_reader_helper.cpp‎

Lines changed: 84 additions & 151 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
#include <arrow/array.h>
2121
#include <arrow/builder.h>
22+
#include <arrow/c/bridge.h>
2223
#include <arrow/extension_type.h>
2324
#include <arrow/type.h>
2425
#include <arrow/util/key_value_metadata.h>
@@ -272,26 +273,6 @@ Status arrow_field_to_doris_type(const std::shared_ptr<arrow::Field>& field,
272273
}
273274
}
274275

275-
// Determine whether a field subtree contains values that require Lance normalization.
276-
Status field_requires_lance_normalization(const std::shared_ptr<arrow::Field>& field,
277-
bool* requires_normalization) {
278-
DORIS_CHECK(field != nullptr);
279-
DORIS_CHECK(requires_normalization != nullptr);
280-
281-
LanceExtensionKind extension_kind;
282-
std::shared_ptr<arrow::DataType> storage_type;
283-
RETURN_IF_ERROR(get_lance_extension(field, &extension_kind, &storage_type));
284-
bool required = extension_kind == LanceExtensionKind::BFLOAT16 ||
285-
field->type()->id() == arrow::Type::EXTENSION;
286-
for (const auto& child : storage_type->fields()) {
287-
bool child_required = false;
288-
RETURN_IF_ERROR(field_requires_lance_normalization(child, &child_required));
289-
required |= child_required;
290-
}
291-
*requires_normalization = required;
292-
return Status::OK();
293-
}
294-
295276
// Widen little-endian Lance BFloat16 values to Arrow Float32 without precision loss.
296277
Status convert_bfloat16_array(const std::shared_ptr<arrow::Array>& array,
297278
arrow::MemoryPool* memory_pool,
@@ -373,89 +354,6 @@ Status set_lance_nested_type(std::string_view field_name,
373354
return Status::OK();
374355
}
375356

376-
// Check whether an Arrow type tree contains a registered extension wrapper.
377-
bool type_contains_registered_extension(const std::shared_ptr<arrow::DataType>& type) {
378-
if (type->id() == arrow::Type::EXTENSION) {
379-
return true;
380-
}
381-
for (const auto& field : type->fields()) {
382-
if (type_contains_registered_extension(field->type())) {
383-
return true;
384-
}
385-
}
386-
return false;
387-
}
388-
389-
// Remove registered ExtensionArray wrappers only along extension-bearing branches.
390-
Status unwrap_lance_extension_arrays(const std::shared_ptr<arrow::DataType>& expected_type,
391-
const std::shared_ptr<arrow::Array>& array,
392-
std::shared_ptr<arrow::Array>* unwrapped) {
393-
DORIS_CHECK(expected_type != nullptr);
394-
DORIS_CHECK(array != nullptr);
395-
DORIS_CHECK(unwrapped != nullptr);
396-
397-
auto storage_array = array;
398-
auto expected_storage_type = expected_type;
399-
if (expected_type->id() == arrow::Type::EXTENSION) {
400-
const auto extension_type = std::dynamic_pointer_cast<arrow::ExtensionType>(expected_type);
401-
if (extension_type == nullptr) {
402-
return Status::InvalidArgument("invalid expected Arrow extension type {}",
403-
expected_type->ToString());
404-
}
405-
expected_storage_type = extension_type->storage_type();
406-
}
407-
if (array->type_id() == arrow::Type::EXTENSION) {
408-
const auto extension_array = std::dynamic_pointer_cast<arrow::ExtensionArray>(array);
409-
if (extension_array == nullptr) {
410-
return Status::InvalidArgument("invalid Arrow extension array: {}",
411-
array->type()->ToString());
412-
}
413-
storage_array = extension_array->storage();
414-
}
415-
416-
const auto& child_data = storage_array->data()->child_data;
417-
const auto& child_fields = expected_storage_type->fields();
418-
if (child_data.empty()) {
419-
*unwrapped = std::move(storage_array);
420-
return Status::OK();
421-
}
422-
if (child_fields.size() != child_data.size()) {
423-
return Status::InvalidArgument(
424-
"Arrow array type {} has {} child fields but its data has {} children",
425-
storage_array->type()->ToString(), child_fields.size(), child_data.size());
426-
}
427-
428-
std::shared_ptr<arrow::ArrayData> unwrapped_data;
429-
arrow::FieldVector unwrapped_fields;
430-
for (size_t child_idx = 0; child_idx < child_data.size(); ++child_idx) {
431-
if (!type_contains_registered_extension(child_fields[child_idx]->type())) {
432-
continue;
433-
}
434-
auto child_array = arrow::MakeArray(child_data[child_idx]);
435-
std::shared_ptr<arrow::Array> unwrapped_child;
436-
RETURN_IF_ERROR(unwrap_lance_extension_arrays(child_fields[child_idx]->type(), child_array,
437-
&unwrapped_child));
438-
if (unwrapped_child.get() == child_array.get()) {
439-
continue;
440-
}
441-
if (unwrapped_data == nullptr) {
442-
unwrapped_data = storage_array->data()->Copy();
443-
unwrapped_fields = storage_array->type()->fields();
444-
}
445-
unwrapped_data->child_data[child_idx] = unwrapped_child->data();
446-
unwrapped_fields[child_idx] =
447-
unwrapped_fields[child_idx]->WithType(unwrapped_child->type());
448-
}
449-
if (unwrapped_data == nullptr) {
450-
*unwrapped = std::move(storage_array);
451-
return Status::OK();
452-
}
453-
RETURN_IF_ERROR(
454-
set_lance_nested_type("", storage_array->type(), unwrapped_fields, &unwrapped_data));
455-
*unwrapped = arrow::MakeArray(std::move(unwrapped_data));
456-
return Status::OK();
457-
}
458-
459357
// Materialize the visible range into an offset-zero Arrow array for Doris SerDes.
460358
Status compact_lance_array(const std::shared_ptr<arrow::Array>& array,
461359
arrow::MemoryPool* memory_pool,
@@ -575,78 +473,92 @@ Status compact_lance_array_if_needed(const std::shared_ptr<arrow::Array>& array,
575473

576474
} // namespace
577475

578-
// Normalize Lance extensions and materialize sliced arrays for Doris Arrow SerDes.
579-
Status normalize_lance_arrow_array(const std::shared_ptr<arrow::Field>& field,
580-
const std::shared_ptr<arrow::Array>& array,
581-
arrow::MemoryPool* memory_pool,
582-
std::shared_ptr<arrow::Array>* normalized) {
476+
Status LanceArrowArrayNormalizer::create(const std::shared_ptr<arrow::Field>& field,
477+
LanceArrowArrayNormalizer* normalizer) {
583478
DORIS_CHECK(field != nullptr);
584-
DORIS_CHECK(array != nullptr);
585-
DORIS_CHECK(memory_pool != nullptr);
586-
DORIS_CHECK(normalized != nullptr);
479+
DORIS_CHECK(normalizer != nullptr);
587480

588481
LanceExtensionKind extension_kind;
589482
std::shared_ptr<arrow::DataType> storage_type;
590483
RETURN_IF_ERROR(get_lance_extension(field, &extension_kind, &storage_type));
591484

485+
LanceArrowArrayNormalizer result;
486+
result._field_name = field->name();
487+
result._storage_type = std::move(storage_type);
488+
result._unwrap_registered_extension = field->type()->id() == arrow::Type::EXTENSION;
489+
result._convert_bfloat16 = extension_kind == LanceExtensionKind::BFLOAT16;
490+
result._requires_special_handling =
491+
result._unwrap_registered_extension || result._convert_bfloat16;
492+
result._child_normalizers.reserve(result._storage_type->num_fields());
493+
for (const auto& child_field : result._storage_type->fields()) {
494+
LanceArrowArrayNormalizer child_normalizer;
495+
RETURN_IF_ERROR(create(child_field, &child_normalizer));
496+
result._requires_special_handling |= child_normalizer._requires_special_handling;
497+
result._child_normalizers.emplace_back(std::move(child_normalizer));
498+
}
499+
*normalizer = std::move(result);
500+
return Status::OK();
501+
}
502+
503+
Status LanceArrowArrayNormalizer::normalize_for_doris(
504+
const std::shared_ptr<arrow::Array>& array, arrow::MemoryPool* memory_pool,
505+
std::shared_ptr<arrow::Array>* normalized) const {
506+
DORIS_CHECK(array != nullptr);
507+
DORIS_CHECK(memory_pool != nullptr);
508+
DORIS_CHECK(normalized != nullptr);
509+
510+
// The schema binding already resolved extension metadata and nested special types. For common
511+
// arrays, runtime work is limited to checking whether the visible slice needs compaction.
512+
if (!_requires_special_handling) {
513+
return compact_lance_array_if_needed(array, memory_pool, normalized);
514+
}
515+
592516
auto storage_array = array;
593-
if (type_contains_registered_extension(field->type())) {
594-
std::shared_ptr<arrow::Array> unwrapped_array;
595-
RETURN_IF_ERROR(unwrap_lance_extension_arrays(field->type(), array, &unwrapped_array));
596-
storage_array = std::move(unwrapped_array);
517+
if (_unwrap_registered_extension && array->type_id() == arrow::Type::EXTENSION) {
518+
const auto extension_array = std::dynamic_pointer_cast<arrow::ExtensionArray>(array);
519+
if (extension_array == nullptr) {
520+
return Status::InvalidArgument("invalid Arrow extension array: {}",
521+
array->type()->ToString());
522+
}
523+
storage_array = extension_array->storage();
597524
}
598-
if (storage_array->type_id() != storage_type->id()) {
525+
// Extension and BFloat16 arrays are first converted to their physical storage representation;
526+
// nested children are then normalized recursively before Doris reads the result.
527+
if (storage_array->type_id() != _storage_type->id()) {
599528
return Status::InvalidArgument(
600-
"Lance field '{}' storage type {} does not match array type {}", field->name(),
601-
storage_type->ToString(), storage_array->type()->ToString());
529+
"Lance field '{}' storage type {} does not match array type {}", _field_name,
530+
_storage_type->ToString(), storage_array->type()->ToString());
602531
}
603-
if (extension_kind == LanceExtensionKind::BFLOAT16) {
532+
if (_convert_bfloat16) {
604533
return convert_bfloat16_array(storage_array, memory_pool, normalized);
605534
}
606535

607536
std::shared_ptr<arrow::Array> compacted_array;
608537
RETURN_IF_ERROR(compact_lance_array_if_needed(storage_array, memory_pool, &compacted_array));
609538
storage_array = std::move(compacted_array);
610-
611-
const auto& child_fields = storage_type->fields();
612-
const auto& child_data = storage_array->data()->child_data;
613-
if (child_fields.empty()) {
539+
if (_child_normalizers.empty()) {
614540
*normalized = std::move(storage_array);
615541
return Status::OK();
616542
}
617-
if (child_fields.size() != child_data.size()) {
543+
544+
const auto& child_data = storage_array->data()->child_data;
545+
if (_child_normalizers.size() != child_data.size()) {
618546
return Status::InvalidArgument(
619547
"Lance field '{}' has {} child fields but its Arrow array has {} children",
620-
field->name(), child_fields.size(), child_data.size());
621-
}
622-
623-
bool requires_normalization = false;
624-
for (const auto& child_field : child_fields) {
625-
bool child_required = false;
626-
RETURN_IF_ERROR(field_requires_lance_normalization(child_field, &child_required));
627-
if (child_required) {
628-
requires_normalization = true;
629-
break;
630-
}
631-
}
632-
if (!requires_normalization) {
633-
*normalized = std::move(storage_array);
634-
return Status::OK();
548+
_field_name, _child_normalizers.size(), child_data.size());
635549
}
636550

637551
arrow::FieldVector normalized_fields;
638552
std::shared_ptr<arrow::ArrayData> normalized_data;
639-
for (size_t child_idx = 0; child_idx < child_fields.size(); ++child_idx) {
640-
bool child_required = false;
641-
RETURN_IF_ERROR(
642-
field_requires_lance_normalization(child_fields[child_idx], &child_required));
643-
if (!child_required) {
553+
for (size_t child_idx = 0; child_idx < _child_normalizers.size(); ++child_idx) {
554+
const auto& child_normalizer = _child_normalizers[child_idx];
555+
if (!child_normalizer._requires_special_handling) {
644556
continue;
645557
}
646-
auto child_array = arrow::MakeArray(storage_array->data()->child_data[child_idx]);
558+
auto child_array = arrow::MakeArray(child_data[child_idx]);
647559
std::shared_ptr<arrow::Array> normalized_child;
648-
RETURN_IF_ERROR(normalize_lance_arrow_array(child_fields[child_idx], child_array,
649-
memory_pool, &normalized_child));
560+
RETURN_IF_ERROR(
561+
child_normalizer.normalize_for_doris(child_array, memory_pool, &normalized_child));
650562
if (normalized_child.get() == child_array.get()) {
651563
continue;
652564
}
@@ -663,7 +575,7 @@ Status normalize_lance_arrow_array(const std::shared_ptr<arrow::Field>& field,
663575
return Status::OK();
664576
}
665577

666-
RETURN_IF_ERROR(set_lance_nested_type(field->name(), storage_array->type(), normalized_fields,
578+
RETURN_IF_ERROR(set_lance_nested_type(_field_name, storage_array->type(), normalized_fields,
667579
&normalized_data));
668580
*normalized = arrow::MakeArray(std::move(normalized_data));
669581
return Status::OK();
@@ -675,12 +587,33 @@ Status normalize_lance_arrow_array_for_test(const std::shared_ptr<arrow::Field>&
675587
const std::shared_ptr<arrow::Array>& array,
676588
std::shared_ptr<arrow::Array>* normalized,
677589
arrow::MemoryPool* memory_pool) {
678-
return normalize_lance_arrow_array(
679-
field, array, memory_pool != nullptr ? memory_pool : arrow::default_memory_pool(),
680-
normalized);
590+
LanceArrowArrayNormalizer normalizer;
591+
RETURN_IF_ERROR(LanceArrowArrayNormalizer::create(field, &normalizer));
592+
return normalizer.normalize_for_doris(
593+
array, memory_pool != nullptr ? memory_pool : arrow::default_memory_pool(), normalized);
681594
}
682595
#endif
683596

597+
Status import_lance_dataset_schema(LanceDataset* dataset, std::shared_ptr<arrow::Schema>* schema) {
598+
DORIS_CHECK(dataset != nullptr);
599+
DORIS_CHECK(schema != nullptr);
600+
601+
ArrowSchema arrow_schema {};
602+
if (lance_dataset_schema(dataset, &arrow_schema) != 0) {
603+
return lance_error("get Lance dataset schema");
604+
}
605+
auto imported_schema = arrow::ImportSchema(&arrow_schema);
606+
if (!imported_schema.ok()) {
607+
if (arrow_schema.release != nullptr) {
608+
arrow_schema.release(&arrow_schema);
609+
}
610+
return Status::InternalError("import Lance Arrow schema failed: {}",
611+
imported_schema.status().message());
612+
}
613+
*schema = std::move(imported_schema).ValueUnsafe();
614+
return Status::OK();
615+
}
616+
684617
void LanceDatasetDeleter::operator()(LanceDataset* dataset) const {
685618
lance_dataset_close(dataset);
686619
}

‎be/src/format_v2/lance/lance_reader_helper.h‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ struct LanceScanner;
3434

3535
namespace arrow {
3636
class Array;
37+
class DataType;
3738
class Field;
3839
class MemoryPool;
3940
class Schema;
@@ -60,17 +61,37 @@ struct LanceBatchDeleter {
6061

6162
size_t lance_vector_element_width(TVectorElementType::type type);
6263

64+
// Import the physical Arrow schema owned by a Lance dataset. The caller owns the resulting
65+
// shared schema and may cache it for operations that need physical Arrow types.
66+
Status import_lance_dataset_schema(LanceDataset* dataset, std::shared_ptr<arrow::Schema>* schema);
67+
6368
// Validate and convert the fragment and index-segment identifiers carried by the FE into the
6469
// unsigned and packed representations expected by lance-c.
6570
Status parse_fragment_ids(const TLanceFileDesc& lance_params, std::vector<uint64_t>* fragment_ids);
6671
Status parse_index_segment_uuids(const TLanceFileDesc& lance_params,
6772
std::vector<uint8_t>* segment_uuids, size_t* segment_count);
6873

69-
// Normalize Lance extension arrays into Arrow arrays supported by Doris.
70-
Status normalize_lance_arrow_array(const std::shared_ptr<arrow::Field>& field,
71-
const std::shared_ptr<arrow::Array>& array,
72-
arrow::MemoryPool* memory_pool,
73-
std::shared_ptr<arrow::Array>* normalized);
74+
// Resolves one Arrow field once when a stream schema is bound. Runtime conversion then reuses this
75+
// plan instead of rescanning extension metadata and nested fields for every array.
76+
class LanceArrowArrayNormalizer {
77+
public:
78+
static Status create(const std::shared_ptr<arrow::Field>& field,
79+
LanceArrowArrayNormalizer* normalizer);
80+
81+
// Return an Arrow array whose physical layout and type can be consumed by Doris SerDes. The
82+
// input is returned unchanged when no compaction or type adaptation is required.
83+
Status normalize_for_doris(const std::shared_ptr<arrow::Array>& array,
84+
arrow::MemoryPool* memory_pool,
85+
std::shared_ptr<arrow::Array>* normalized) const;
86+
87+
private:
88+
std::string _field_name;
89+
std::shared_ptr<arrow::DataType> _storage_type;
90+
std::vector<LanceArrowArrayNormalizer> _child_normalizers;
91+
bool _unwrap_registered_extension = false;
92+
bool _convert_bfloat16 = false;
93+
bool _requires_special_handling = false;
94+
};
7495

7596
#ifdef BE_TEST
7697
// Expose Lance Arrow normalization for allocation-sensitive unit tests.

0 commit comments

Comments
 (0)