1919
2020#include < arrow/array.h>
2121#include < arrow/builder.h>
22+ #include < arrow/c/bridge.h>
2223#include < arrow/extension_type.h>
2324#include < arrow/type.h>
2425#include < arrow/util/key_value_metadata.h>
@@ -272,26 +273,6 @@ Status arrow_field_to_doris_type(const std::shared_ptr<arrow::Field>& field,
272273 }
273274}
274275
275- // Determine whether a field subtree contains values that require Lance normalization.
276- Status field_requires_lance_normalization (const std::shared_ptr<arrow::Field>& field,
277- bool * requires_normalization) {
278- DORIS_CHECK (field != nullptr );
279- DORIS_CHECK (requires_normalization != nullptr );
280-
281- LanceExtensionKind extension_kind;
282- std::shared_ptr<arrow::DataType> storage_type;
283- RETURN_IF_ERROR (get_lance_extension (field, &extension_kind, &storage_type));
284- bool required = extension_kind == LanceExtensionKind::BFLOAT16 ||
285- field->type ()->id () == arrow::Type::EXTENSION ;
286- for (const auto & child : storage_type->fields ()) {
287- bool child_required = false ;
288- RETURN_IF_ERROR (field_requires_lance_normalization (child, &child_required));
289- required |= child_required;
290- }
291- *requires_normalization = required;
292- return Status::OK ();
293- }
294-
295276// Widen little-endian Lance BFloat16 values to Arrow Float32 without precision loss.
296277Status convert_bfloat16_array (const std::shared_ptr<arrow::Array>& array,
297278 arrow::MemoryPool* memory_pool,
@@ -373,89 +354,6 @@ Status set_lance_nested_type(std::string_view field_name,
373354 return Status::OK ();
374355}
375356
376- // Check whether an Arrow type tree contains a registered extension wrapper.
377- bool type_contains_registered_extension (const std::shared_ptr<arrow::DataType>& type) {
378- if (type->id () == arrow::Type::EXTENSION ) {
379- return true ;
380- }
381- for (const auto & field : type->fields ()) {
382- if (type_contains_registered_extension (field->type ())) {
383- return true ;
384- }
385- }
386- return false ;
387- }
388-
389- // Remove registered ExtensionArray wrappers only along extension-bearing branches.
390- Status unwrap_lance_extension_arrays (const std::shared_ptr<arrow::DataType>& expected_type,
391- const std::shared_ptr<arrow::Array>& array,
392- std::shared_ptr<arrow::Array>* unwrapped) {
393- DORIS_CHECK (expected_type != nullptr );
394- DORIS_CHECK (array != nullptr );
395- DORIS_CHECK (unwrapped != nullptr );
396-
397- auto storage_array = array;
398- auto expected_storage_type = expected_type;
399- if (expected_type->id () == arrow::Type::EXTENSION ) {
400- const auto extension_type = std::dynamic_pointer_cast<arrow::ExtensionType>(expected_type);
401- if (extension_type == nullptr ) {
402- return Status::InvalidArgument (" invalid expected Arrow extension type {}" ,
403- expected_type->ToString ());
404- }
405- expected_storage_type = extension_type->storage_type ();
406- }
407- if (array->type_id () == arrow::Type::EXTENSION ) {
408- const auto extension_array = std::dynamic_pointer_cast<arrow::ExtensionArray>(array);
409- if (extension_array == nullptr ) {
410- return Status::InvalidArgument (" invalid Arrow extension array: {}" ,
411- array->type ()->ToString ());
412- }
413- storage_array = extension_array->storage ();
414- }
415-
416- const auto & child_data = storage_array->data ()->child_data ;
417- const auto & child_fields = expected_storage_type->fields ();
418- if (child_data.empty ()) {
419- *unwrapped = std::move (storage_array);
420- return Status::OK ();
421- }
422- if (child_fields.size () != child_data.size ()) {
423- return Status::InvalidArgument (
424- " Arrow array type {} has {} child fields but its data has {} children" ,
425- storage_array->type ()->ToString (), child_fields.size (), child_data.size ());
426- }
427-
428- std::shared_ptr<arrow::ArrayData> unwrapped_data;
429- arrow::FieldVector unwrapped_fields;
430- for (size_t child_idx = 0 ; child_idx < child_data.size (); ++child_idx) {
431- if (!type_contains_registered_extension (child_fields[child_idx]->type ())) {
432- continue ;
433- }
434- auto child_array = arrow::MakeArray (child_data[child_idx]);
435- std::shared_ptr<arrow::Array> unwrapped_child;
436- RETURN_IF_ERROR (unwrap_lance_extension_arrays (child_fields[child_idx]->type (), child_array,
437- &unwrapped_child));
438- if (unwrapped_child.get () == child_array.get ()) {
439- continue ;
440- }
441- if (unwrapped_data == nullptr ) {
442- unwrapped_data = storage_array->data ()->Copy ();
443- unwrapped_fields = storage_array->type ()->fields ();
444- }
445- unwrapped_data->child_data [child_idx] = unwrapped_child->data ();
446- unwrapped_fields[child_idx] =
447- unwrapped_fields[child_idx]->WithType (unwrapped_child->type ());
448- }
449- if (unwrapped_data == nullptr ) {
450- *unwrapped = std::move (storage_array);
451- return Status::OK ();
452- }
453- RETURN_IF_ERROR (
454- set_lance_nested_type (" " , storage_array->type (), unwrapped_fields, &unwrapped_data));
455- *unwrapped = arrow::MakeArray (std::move (unwrapped_data));
456- return Status::OK ();
457- }
458-
459357// Materialize the visible range into an offset-zero Arrow array for Doris SerDes.
460358Status compact_lance_array (const std::shared_ptr<arrow::Array>& array,
461359 arrow::MemoryPool* memory_pool,
@@ -575,78 +473,92 @@ Status compact_lance_array_if_needed(const std::shared_ptr<arrow::Array>& array,
575473
576474} // namespace
577475
578- // Normalize Lance extensions and materialize sliced arrays for Doris Arrow SerDes.
579- Status normalize_lance_arrow_array (const std::shared_ptr<arrow::Field>& field,
580- const std::shared_ptr<arrow::Array>& array,
581- arrow::MemoryPool* memory_pool,
582- std::shared_ptr<arrow::Array>* normalized) {
476+ Status LanceArrowArrayNormalizer::create (const std::shared_ptr<arrow::Field>& field,
477+ LanceArrowArrayNormalizer* normalizer) {
583478 DORIS_CHECK (field != nullptr );
584- DORIS_CHECK (array != nullptr );
585- DORIS_CHECK (memory_pool != nullptr );
586- DORIS_CHECK (normalized != nullptr );
479+ DORIS_CHECK (normalizer != nullptr );
587480
588481 LanceExtensionKind extension_kind;
589482 std::shared_ptr<arrow::DataType> storage_type;
590483 RETURN_IF_ERROR (get_lance_extension (field, &extension_kind, &storage_type));
591484
485+ LanceArrowArrayNormalizer result;
486+ result._field_name = field->name ();
487+ result._storage_type = std::move (storage_type);
488+ result._unwrap_registered_extension = field->type ()->id () == arrow::Type::EXTENSION ;
489+ result._convert_bfloat16 = extension_kind == LanceExtensionKind::BFLOAT16 ;
490+ result._requires_special_handling =
491+ result._unwrap_registered_extension || result._convert_bfloat16 ;
492+ result._child_normalizers .reserve (result._storage_type ->num_fields ());
493+ for (const auto & child_field : result._storage_type ->fields ()) {
494+ LanceArrowArrayNormalizer child_normalizer;
495+ RETURN_IF_ERROR (create (child_field, &child_normalizer));
496+ result._requires_special_handling |= child_normalizer._requires_special_handling ;
497+ result._child_normalizers .emplace_back (std::move (child_normalizer));
498+ }
499+ *normalizer = std::move (result);
500+ return Status::OK ();
501+ }
502+
503+ Status LanceArrowArrayNormalizer::normalize_for_doris (
504+ const std::shared_ptr<arrow::Array>& array, arrow::MemoryPool* memory_pool,
505+ std::shared_ptr<arrow::Array>* normalized) const {
506+ DORIS_CHECK (array != nullptr );
507+ DORIS_CHECK (memory_pool != nullptr );
508+ DORIS_CHECK (normalized != nullptr );
509+
510+ // The schema binding already resolved extension metadata and nested special types. For common
511+ // arrays, runtime work is limited to checking whether the visible slice needs compaction.
512+ if (!_requires_special_handling) {
513+ return compact_lance_array_if_needed (array, memory_pool, normalized);
514+ }
515+
592516 auto storage_array = array;
593- if (type_contains_registered_extension (field->type ())) {
594- std::shared_ptr<arrow::Array> unwrapped_array;
595- RETURN_IF_ERROR (unwrap_lance_extension_arrays (field->type (), array, &unwrapped_array));
596- storage_array = std::move (unwrapped_array);
517+ if (_unwrap_registered_extension && array->type_id () == arrow::Type::EXTENSION ) {
518+ const auto extension_array = std::dynamic_pointer_cast<arrow::ExtensionArray>(array);
519+ if (extension_array == nullptr ) {
520+ return Status::InvalidArgument (" invalid Arrow extension array: {}" ,
521+ array->type ()->ToString ());
522+ }
523+ storage_array = extension_array->storage ();
597524 }
598- if (storage_array->type_id () != storage_type->id ()) {
525+ // Extension and BFloat16 arrays are first converted to their physical storage representation;
526+ // nested children are then normalized recursively before Doris reads the result.
527+ if (storage_array->type_id () != _storage_type->id ()) {
599528 return Status::InvalidArgument (
600- " Lance field '{}' storage type {} does not match array type {}" , field-> name () ,
601- storage_type ->ToString (), storage_array->type ()->ToString ());
529+ " Lance field '{}' storage type {} does not match array type {}" , _field_name ,
530+ _storage_type ->ToString (), storage_array->type ()->ToString ());
602531 }
603- if (extension_kind == LanceExtensionKind:: BFLOAT16 ) {
532+ if (_convert_bfloat16 ) {
604533 return convert_bfloat16_array (storage_array, memory_pool, normalized);
605534 }
606535
607536 std::shared_ptr<arrow::Array> compacted_array;
608537 RETURN_IF_ERROR (compact_lance_array_if_needed (storage_array, memory_pool, &compacted_array));
609538 storage_array = std::move (compacted_array);
610-
611- const auto & child_fields = storage_type->fields ();
612- const auto & child_data = storage_array->data ()->child_data ;
613- if (child_fields.empty ()) {
539+ if (_child_normalizers.empty ()) {
614540 *normalized = std::move (storage_array);
615541 return Status::OK ();
616542 }
617- if (child_fields.size () != child_data.size ()) {
543+
544+ const auto & child_data = storage_array->data ()->child_data ;
545+ if (_child_normalizers.size () != child_data.size ()) {
618546 return Status::InvalidArgument (
619547 " Lance field '{}' has {} child fields but its Arrow array has {} children" ,
620- field->name (), child_fields.size (), child_data.size ());
621- }
622-
623- bool requires_normalization = false ;
624- for (const auto & child_field : child_fields) {
625- bool child_required = false ;
626- RETURN_IF_ERROR (field_requires_lance_normalization (child_field, &child_required));
627- if (child_required) {
628- requires_normalization = true ;
629- break ;
630- }
631- }
632- if (!requires_normalization) {
633- *normalized = std::move (storage_array);
634- return Status::OK ();
548+ _field_name, _child_normalizers.size (), child_data.size ());
635549 }
636550
637551 arrow::FieldVector normalized_fields;
638552 std::shared_ptr<arrow::ArrayData> normalized_data;
639- for (size_t child_idx = 0 ; child_idx < child_fields.size (); ++child_idx) {
640- bool child_required = false ;
641- RETURN_IF_ERROR (
642- field_requires_lance_normalization (child_fields[child_idx], &child_required));
643- if (!child_required) {
553+ for (size_t child_idx = 0 ; child_idx < _child_normalizers.size (); ++child_idx) {
554+ const auto & child_normalizer = _child_normalizers[child_idx];
555+ if (!child_normalizer._requires_special_handling ) {
644556 continue ;
645557 }
646- auto child_array = arrow::MakeArray (storage_array-> data ()-> child_data [child_idx]);
558+ auto child_array = arrow::MakeArray (child_data[child_idx]);
647559 std::shared_ptr<arrow::Array> normalized_child;
648- RETURN_IF_ERROR (normalize_lance_arrow_array (child_fields[child_idx], child_array,
649- memory_pool, &normalized_child));
560+ RETURN_IF_ERROR (
561+ child_normalizer. normalize_for_doris (child_array, memory_pool, &normalized_child));
650562 if (normalized_child.get () == child_array.get ()) {
651563 continue ;
652564 }
@@ -663,7 +575,7 @@ Status normalize_lance_arrow_array(const std::shared_ptr<arrow::Field>& field,
663575 return Status::OK ();
664576 }
665577
666- RETURN_IF_ERROR (set_lance_nested_type (field-> name () , storage_array->type (), normalized_fields,
578+ RETURN_IF_ERROR (set_lance_nested_type (_field_name , storage_array->type (), normalized_fields,
667579 &normalized_data));
668580 *normalized = arrow::MakeArray (std::move (normalized_data));
669581 return Status::OK ();
@@ -675,12 +587,33 @@ Status normalize_lance_arrow_array_for_test(const std::shared_ptr<arrow::Field>&
675587 const std::shared_ptr<arrow::Array>& array,
676588 std::shared_ptr<arrow::Array>* normalized,
677589 arrow::MemoryPool* memory_pool) {
678- return normalize_lance_arrow_array (
679- field, array, memory_pool != nullptr ? memory_pool : arrow::default_memory_pool (),
680- normalized);
590+ LanceArrowArrayNormalizer normalizer;
591+ RETURN_IF_ERROR (LanceArrowArrayNormalizer::create (field, &normalizer));
592+ return normalizer.normalize_for_doris (
593+ array, memory_pool != nullptr ? memory_pool : arrow::default_memory_pool (), normalized);
681594}
682595#endif
683596
597+ Status import_lance_dataset_schema (LanceDataset* dataset, std::shared_ptr<arrow::Schema>* schema) {
598+ DORIS_CHECK (dataset != nullptr );
599+ DORIS_CHECK (schema != nullptr );
600+
601+ ArrowSchema arrow_schema {};
602+ if (lance_dataset_schema (dataset, &arrow_schema) != 0 ) {
603+ return lance_error (" get Lance dataset schema" );
604+ }
605+ auto imported_schema = arrow::ImportSchema (&arrow_schema);
606+ if (!imported_schema.ok ()) {
607+ if (arrow_schema.release != nullptr ) {
608+ arrow_schema.release (&arrow_schema);
609+ }
610+ return Status::InternalError (" import Lance Arrow schema failed: {}" ,
611+ imported_schema.status ().message ());
612+ }
613+ *schema = std::move (imported_schema).ValueUnsafe ();
614+ return Status::OK ();
615+ }
616+
684617void LanceDatasetDeleter::operator ()(LanceDataset* dataset) const {
685618 lance_dataset_close (dataset);
686619}
0 commit comments