|
26 | 26 | #include "core/column/column.h" |
27 | 27 | #include "core/column/column_const.h" |
28 | 28 | #include "core/column/column_map.h" |
| 29 | +#include "core/data_type/data_type_map.h" |
29 | 30 | #include "core/data_type_serde/arrow_validation.h" |
30 | 31 | #include "core/data_type_serde/complex_type_deserialize_util.h" |
31 | 32 | #include "core/data_type_serde/orc_serde_utils.h" |
@@ -440,6 +441,108 @@ Status DataTypeMapSerDe::write_column_to_arrow(const IColumn& column, const Null |
440 | 441 | return Status::OK(); |
441 | 442 | } |
442 | 443 |
|
| 444 | +namespace { |
| 445 | + |
| 446 | +template <typename WriteKey, typename WriteValue> |
| 447 | +Status write_map_column_to_target(const IColumn& column, const NullMap* null_map, |
| 448 | + arrow::ArrayBuilder* array_builder, int64_t start, int64_t end, |
| 449 | + WriteKey&& write_key, WriteValue&& write_value) { |
| 450 | + auto& builder = assert_cast<arrow::MapBuilder&>(*array_builder); |
| 451 | + const auto& map_column = assert_cast<const ColumnMap&>(column); |
| 452 | + const IColumn& nested_keys_column = map_column.get_keys(); |
| 453 | + const IColumn& nested_values_column = map_column.get_values(); |
| 454 | + DCHECK(nested_keys_column.is_nullable()); |
| 455 | + DCHECK(nested_values_column.is_nullable()); |
| 456 | + const auto* keys_nullmap_data = |
| 457 | + check_and_get_column<ColumnNullable>(nested_keys_column)->get_null_map_data().data(); |
| 458 | + const auto& offsets = map_column.get_offsets(); |
| 459 | + auto* key_builder = builder.key_builder(); |
| 460 | + auto* value_builder = builder.item_builder(); |
| 461 | + |
| 462 | + for (size_t row = start; row < end; ++row) { |
| 463 | + if (null_map != nullptr && (*null_map)[row]) { |
| 464 | + RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column, *array_builder)); |
| 465 | + continue; |
| 466 | + } |
| 467 | + if (simd::contain_one(keys_nullmap_data + offsets[row - 1], |
| 468 | + offsets[row] - offsets[row - 1])) { |
| 469 | + return Status::Error(ErrorCode::INVALID_ARGUMENT, |
| 470 | + "Can not write null value of map key to arrow."); |
| 471 | + } |
| 472 | + RETURN_IF_ERROR(checkArrowStatus(builder.Append(), column, *array_builder)); |
| 473 | + RETURN_IF_ERROR(write_key(nested_keys_column, key_builder, offsets[row - 1], offsets[row])); |
| 474 | + RETURN_IF_ERROR( |
| 475 | + write_value(nested_values_column, value_builder, offsets[row - 1], offsets[row])); |
| 476 | + } |
| 477 | + return Status::OK(); |
| 478 | +} |
| 479 | + |
| 480 | +} // namespace |
| 481 | + |
| 482 | +Status DataTypeMapSerDe::write_column_to_paimon_arrow(const std::shared_ptr<const IDataType>& type, |
| 483 | + const IColumn& column, |
| 484 | + const NullMap* null_map, |
| 485 | + const std::shared_ptr<arrow::Field>& field, |
| 486 | + arrow::ArrayBuilder* array_builder, |
| 487 | + int64_t start, int64_t end, |
| 488 | + const cctz::time_zone& ctz) const { |
| 489 | + // Reject an incompatible target before casting its nested schema or builder. |
| 490 | + if (field->type()->id() != arrow::Type::MAP || |
| 491 | + array_builder->type()->id() != arrow::Type::MAP) { |
| 492 | + return Status::InvalidArgument("Paimon map writer requires an Arrow map field"); |
| 493 | + } |
| 494 | + const auto& map_type = assert_cast<const DataTypeMap&>(*type); |
| 495 | + const auto& arrow_map_type = assert_cast<const arrow::MapType&>(*field->type()); |
| 496 | + const auto& key_field = arrow_map_type.key_field(); |
| 497 | + const auto& value_field = arrow_map_type.item_field(); |
| 498 | + return write_map_column_to_target( |
| 499 | + column, null_map, array_builder, start, end, |
| 500 | + [&](const IColumn& nested_keys, arrow::ArrayBuilder* key_builder, int64_t nested_start, |
| 501 | + int64_t nested_end) { |
| 502 | + return key_serde->write_column_to_paimon_arrow(map_type.get_key_type(), nested_keys, |
| 503 | + nullptr, key_field, key_builder, |
| 504 | + nested_start, nested_end, ctz); |
| 505 | + }, |
| 506 | + [&](const IColumn& nested_values, arrow::ArrayBuilder* value_builder, |
| 507 | + int64_t nested_start, int64_t nested_end) { |
| 508 | + return value_serde->write_column_to_paimon_arrow( |
| 509 | + map_type.get_value_type(), nested_values, nullptr, value_field, |
| 510 | + value_builder, nested_start, nested_end, ctz); |
| 511 | + }); |
| 512 | +} |
| 513 | + |
| 514 | +Status DataTypeMapSerDe::write_column_to_iceberg_arrow(const std::shared_ptr<const IDataType>& type, |
| 515 | + const IColumn& column, |
| 516 | + const NullMap* null_map, |
| 517 | + const std::shared_ptr<arrow::Field>& field, |
| 518 | + arrow::ArrayBuilder* array_builder, |
| 519 | + int64_t start, int64_t end, |
| 520 | + const cctz::time_zone& ctz) const { |
| 521 | + // Reject an incompatible target before casting its nested schema or builder. |
| 522 | + if (field->type()->id() != arrow::Type::MAP || |
| 523 | + array_builder->type()->id() != arrow::Type::MAP) { |
| 524 | + return Status::InvalidArgument("Iceberg map writer requires an Arrow map field"); |
| 525 | + } |
| 526 | + const auto& map_type = assert_cast<const DataTypeMap&>(*type); |
| 527 | + const auto& arrow_map_type = assert_cast<const arrow::MapType&>(*field->type()); |
| 528 | + const auto& key_field = arrow_map_type.key_field(); |
| 529 | + const auto& value_field = arrow_map_type.item_field(); |
| 530 | + return write_map_column_to_target( |
| 531 | + column, null_map, array_builder, start, end, |
| 532 | + [&](const IColumn& nested_keys, arrow::ArrayBuilder* key_builder, int64_t nested_start, |
| 533 | + int64_t nested_end) { |
| 534 | + return key_serde->write_column_to_iceberg_arrow( |
| 535 | + map_type.get_key_type(), nested_keys, nullptr, key_field, key_builder, |
| 536 | + nested_start, nested_end, ctz); |
| 537 | + }, |
| 538 | + [&](const IColumn& nested_values, arrow::ArrayBuilder* value_builder, |
| 539 | + int64_t nested_start, int64_t nested_end) { |
| 540 | + return value_serde->write_column_to_iceberg_arrow( |
| 541 | + map_type.get_value_type(), nested_values, nullptr, value_field, |
| 542 | + value_builder, nested_start, nested_end, ctz); |
| 543 | + }); |
| 544 | +} |
| 545 | + |
443 | 546 | Status DataTypeMapSerDe::read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, |
444 | 547 | int64_t start, int64_t end, |
445 | 548 | const cctz::time_zone& ctz) const { |
|
0 commit comments