|
18 | 18 | #include "format/table/iceberg/arrow_schema_util.h" |
19 | 19 |
|
20 | 20 | #include <arrow/api.h> |
| 21 | +#include <arrow/extension/parquet_variant.h> |
21 | 22 | #include <arrow/io/api.h> |
22 | 23 | #include <arrow/status.h> |
23 | 24 | #include <arrow/type.h> |
|
28 | 29 | #include <parquet/arrow/writer.h> |
29 | 30 | #include <parquet/schema.h> |
30 | 31 |
|
| 32 | +#include "core/assert_cast.h" |
31 | 33 | #include "core/block/block.h" |
32 | 34 | #include "core/column/column_array.h" |
33 | 35 | #include "core/column/column_nullable.h" |
@@ -358,6 +360,101 @@ TEST(ArrowSchemaUtilTest, test_binary_field_types) { |
358 | 360 | EXPECT_EQ("uuid", fields[3]->metadata()->Get("originalType").ValueUnsafe()); |
359 | 361 | } |
360 | 362 |
|
| 363 | +TEST(ArrowSchemaUtilTest, test_variant_field) { |
| 364 | + std::vector<NestedField> nested_fields; |
| 365 | + nested_fields.emplace_back(true, 21, "payload", std::make_unique<VariantType>(), std::nullopt); |
| 366 | + Schema schema(1, std::move(nested_fields)); |
| 367 | + |
| 368 | + std::vector<std::shared_ptr<arrow::Field>> fields; |
| 369 | + Status st = IcebergArrowSchemaConvertor(schema, "utc").convert_fields(fields); |
| 370 | + ASSERT_TRUE(st.ok()) << st; |
| 371 | + ASSERT_EQ(1, fields.size()); |
| 372 | + EXPECT_EQ("21", fields[0]->metadata()->Get(pfid).ValueUnsafe()); |
| 373 | + ASSERT_EQ(arrow::Type::EXTENSION, fields[0]->type()->id()); |
| 374 | + |
| 375 | + const auto& extension = static_cast<const arrow::ExtensionType&>(*fields[0]->type()); |
| 376 | + EXPECT_EQ("arrow.parquet.variant", extension.extension_name()); |
| 377 | + ASSERT_EQ(arrow::Type::STRUCT, extension.storage_type()->id()); |
| 378 | + ASSERT_EQ(2, extension.storage_type()->num_fields()); |
| 379 | + EXPECT_EQ("metadata", extension.storage_type()->field(0)->name()); |
| 380 | + EXPECT_EQ(arrow::Type::BINARY, extension.storage_type()->field(0)->type()->id()); |
| 381 | + EXPECT_FALSE(extension.storage_type()->field(0)->nullable()); |
| 382 | + EXPECT_FALSE(extension.storage_type()->field(0)->HasMetadata()); |
| 383 | + EXPECT_EQ("value", extension.storage_type()->field(1)->name()); |
| 384 | + EXPECT_EQ(arrow::Type::BINARY, extension.storage_type()->field(1)->type()->id()); |
| 385 | + EXPECT_FALSE(extension.storage_type()->field(1)->nullable()); |
| 386 | + EXPECT_FALSE(extension.storage_type()->field(1)->HasMetadata()); |
| 387 | +} |
| 388 | + |
| 389 | +TEST(ArrowSchemaUtilTest, test_nested_variant_fields) { |
| 390 | + const std::string schema_json = R"({ |
| 391 | + "type": "struct", |
| 392 | + "fields": [ |
| 393 | + { |
| 394 | + "id": 10, |
| 395 | + "name": "info", |
| 396 | + "required": false, |
| 397 | + "type": { |
| 398 | + "type": "struct", |
| 399 | + "fields": [ |
| 400 | + {"id": 11, "name": "payload", "required": false, "type": "variant"} |
| 401 | + ] |
| 402 | + } |
| 403 | + }, |
| 404 | + { |
| 405 | + "id": 20, |
| 406 | + "name": "events", |
| 407 | + "required": false, |
| 408 | + "type": { |
| 409 | + "type": "list", |
| 410 | + "element-id": 21, |
| 411 | + "element": "variant", |
| 412 | + "element-required": false |
| 413 | + } |
| 414 | + }, |
| 415 | + { |
| 416 | + "id": 30, |
| 417 | + "name": "attrs", |
| 418 | + "required": false, |
| 419 | + "type": { |
| 420 | + "type": "map", |
| 421 | + "key-id": 31, |
| 422 | + "key": "string", |
| 423 | + "value-id": 32, |
| 424 | + "value": "variant", |
| 425 | + "value-required": false |
| 426 | + } |
| 427 | + } |
| 428 | + ] |
| 429 | + })"; |
| 430 | + std::unique_ptr<Schema> schema = SchemaParser::from_json(schema_json); |
| 431 | + |
| 432 | + std::vector<std::shared_ptr<arrow::Field>> fields; |
| 433 | + Status status = IcebergArrowSchemaConvertor(*schema, "UTC").convert_fields(fields); |
| 434 | + ASSERT_TRUE(status.ok()) << status; |
| 435 | + ASSERT_EQ(3, fields.size()); |
| 436 | + |
| 437 | + const auto& info = assert_cast<const arrow::StructType&>(*fields[0]->type()); |
| 438 | + ASSERT_EQ(arrow::Type::EXTENSION, info.field(0)->type()->id()); |
| 439 | + EXPECT_EQ("11", info.field(0)->metadata()->Get(pfid).ValueUnsafe()); |
| 440 | + |
| 441 | + const auto& events = assert_cast<const arrow::ListType&>(*fields[1]->type()); |
| 442 | + ASSERT_EQ(arrow::Type::EXTENSION, events.value_type()->id()); |
| 443 | + EXPECT_EQ("21", events.value_field()->metadata()->Get(pfid).ValueUnsafe()); |
| 444 | + |
| 445 | + const auto& attrs = assert_cast<const arrow::MapType&>(*fields[2]->type()); |
| 446 | + ASSERT_EQ(arrow::Type::EXTENSION, attrs.item_type()->id()); |
| 447 | + EXPECT_EQ("32", attrs.item_field()->metadata()->Get(pfid).ValueUnsafe()); |
| 448 | + |
| 449 | + for (const auto& variant_type : |
| 450 | + {info.field(0)->type(), events.value_type(), attrs.item_type()}) { |
| 451 | + const auto& extension = static_cast<const arrow::ExtensionType&>(*variant_type); |
| 452 | + EXPECT_EQ("arrow.parquet.variant", extension.extension_name()); |
| 453 | + EXPECT_EQ("struct<metadata: binary not null, value: binary not null>", |
| 454 | + extension.storage_type()->ToString()); |
| 455 | + } |
| 456 | +} |
| 457 | + |
361 | 458 | TEST(ArrowSchemaUtilTest, test_parquet_filed_id) { |
362 | 459 | std::string test_dir = "ut_dir/test_parquet_filed_id"; |
363 | 460 | Status st; |
|
0 commit comments