|
60 | 60 | #include "core/data_type/data_type_struct.h" |
61 | 61 | #include "core/data_type/data_type_timestamptz.h" |
62 | 62 | #include "core/data_type/data_type_varbinary.h" |
| 63 | +#include "core/data_type/data_type_variant_v2.h" |
63 | 64 | #include "exec/common/endian.h" |
64 | 65 | #include "exec/scan/access_path_parser.h" |
65 | 66 | #include "exprs/runtime_filter_expr.h" |
@@ -1505,6 +1506,108 @@ TEST(IcebergV2ReaderTest, AnnotateConvertsTimestamptzDefaultToSessionTimezone) { |
1505 | 1506 | "2025-01-18 09:02:03.654321"); |
1506 | 1507 | } |
1507 | 1508 |
|
| 1509 | +TEST(IcebergV2ReaderTest, AnnotateVariantInitialDefaultMustBeNull) { |
| 1510 | + const auto variant_type = make_nullable(std::make_shared<DataTypeVariantV2>()); |
| 1511 | + |
| 1512 | + auto optional_variant = |
| 1513 | + external_schema_field("payload", 1, {}, std::nullopt, |
| 1514 | + external_primitive_type(TPrimitiveType::VARIANT), false, true); |
| 1515 | + TFileScanRangeParams valid_scan_params; |
| 1516 | + valid_scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2); |
| 1517 | + valid_scan_params.__set_current_schema_id(100); |
| 1518 | + valid_scan_params.__set_history_schema_info( |
| 1519 | + {external_schema(100, {std::move(optional_variant)})}); |
| 1520 | + |
| 1521 | + ColumnDefinition valid_column; |
| 1522 | + valid_column.name = "payload"; |
| 1523 | + valid_column.type = variant_type; |
| 1524 | + ProjectedColumnBuildContext valid_context {.scan_params = &valid_scan_params}; |
| 1525 | + doris::format::iceberg::IcebergTableReader reader; |
| 1526 | + ASSERT_TRUE(reader.annotate_projected_column(TFileScanSlotInfo(), &valid_context, &valid_column) |
| 1527 | + .ok()); |
| 1528 | + ASSERT_TRUE(valid_column.is_optional.has_value()); |
| 1529 | + EXPECT_TRUE(*valid_column.is_optional); |
| 1530 | + EXPECT_FALSE(valid_column.initial_default_value.has_value()); |
| 1531 | + EXPECT_EQ(valid_column.default_expr, nullptr); |
| 1532 | + ASSERT_TRUE(valid_context.schema_column.has_value()); |
| 1533 | + EXPECT_TRUE(valid_context.schema_column->type->equals(*variant_type)); |
| 1534 | + |
| 1535 | + auto malformed_variant = |
| 1536 | + external_schema_field("payload", 1, {}, "{\"source\":\"malformed\"}", |
| 1537 | + external_primitive_type(TPrimitiveType::VARIANT), false, true); |
| 1538 | + TFileScanRangeParams malformed_scan_params; |
| 1539 | + malformed_scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2); |
| 1540 | + malformed_scan_params.__set_current_schema_id(101); |
| 1541 | + malformed_scan_params.__set_history_schema_info( |
| 1542 | + {external_schema(101, {std::move(malformed_variant)})}); |
| 1543 | + |
| 1544 | + ColumnDefinition malformed_column; |
| 1545 | + malformed_column.name = "payload"; |
| 1546 | + malformed_column.type = variant_type; |
| 1547 | + ProjectedColumnBuildContext malformed_context {.scan_params = &malformed_scan_params}; |
| 1548 | + const auto status = reader.annotate_projected_column(TFileScanSlotInfo(), &malformed_context, |
| 1549 | + &malformed_column); |
| 1550 | + EXPECT_TRUE(status.is<ErrorCode::INVALID_ARGUMENT>()) << status; |
| 1551 | + EXPECT_NE(status.to_string().find("VARIANT initial-default for field 'payload' must be NULL"), |
| 1552 | + std::string::npos) |
| 1553 | + << status; |
| 1554 | +} |
| 1555 | + |
| 1556 | +TEST(IcebergV2ReaderTest, StructInitialDefaultKeepsVariantChildNull) { |
| 1557 | + const auto variant_type = make_nullable(std::make_shared<DataTypeVariantV2>()); |
| 1558 | + const auto struct_type = make_nullable( |
| 1559 | + std::make_shared<DataTypeStruct>(DataTypes {variant_type}, Strings {"payload"})); |
| 1560 | + |
| 1561 | + auto optional_variant = |
| 1562 | + external_schema_field("payload", 2, {}, std::nullopt, |
| 1563 | + external_primitive_type(TPrimitiveType::VARIANT), false, true); |
| 1564 | + auto valid_struct = |
| 1565 | + external_struct_schema_field("event", 1, {std::move(optional_variant)}, true, "{}"); |
| 1566 | + TFileScanRangeParams valid_scan_params; |
| 1567 | + valid_scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2); |
| 1568 | + valid_scan_params.__set_current_schema_id(100); |
| 1569 | + valid_scan_params.__set_history_schema_info({external_schema(100, {std::move(valid_struct)})}); |
| 1570 | + |
| 1571 | + ColumnDefinition valid_column; |
| 1572 | + valid_column.name = "event"; |
| 1573 | + valid_column.type = struct_type; |
| 1574 | + ProjectedColumnBuildContext valid_context {.scan_params = &valid_scan_params}; |
| 1575 | + doris::format::iceberg::IcebergTableReader reader; |
| 1576 | + ASSERT_TRUE(reader.annotate_projected_column(TFileScanSlotInfo(), &valid_context, &valid_column) |
| 1577 | + .ok()); |
| 1578 | + ASSERT_NE(valid_column.default_expr, nullptr); |
| 1579 | + const auto* literal = dynamic_cast<const VLiteral*>(valid_column.default_expr->root().get()); |
| 1580 | + ASSERT_NE(literal, nullptr); |
| 1581 | + EXPECT_TRUE(literal->get_data_type()->equals(*struct_type)); |
| 1582 | + Field value; |
| 1583 | + literal->get_column_ptr()->get(0, value); |
| 1584 | + const auto& struct_fields = value.get<TYPE_STRUCT>(); |
| 1585 | + ASSERT_EQ(struct_fields.size(), 1); |
| 1586 | + EXPECT_TRUE(struct_fields[0].is_null()); |
| 1587 | + |
| 1588 | + auto malformed_variant = |
| 1589 | + external_schema_field("payload", 2, {}, std::nullopt, |
| 1590 | + external_primitive_type(TPrimitiveType::VARIANT), false, true); |
| 1591 | + auto malformed_struct = external_struct_schema_field( |
| 1592 | + "event", 1, {std::move(malformed_variant)}, true, "{\"2\":{\"source\":\"malformed\"}}"); |
| 1593 | + TFileScanRangeParams malformed_scan_params; |
| 1594 | + malformed_scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2); |
| 1595 | + malformed_scan_params.__set_current_schema_id(101); |
| 1596 | + malformed_scan_params.__set_history_schema_info( |
| 1597 | + {external_schema(101, {std::move(malformed_struct)})}); |
| 1598 | + |
| 1599 | + ColumnDefinition malformed_column; |
| 1600 | + malformed_column.name = "event"; |
| 1601 | + malformed_column.type = struct_type; |
| 1602 | + ProjectedColumnBuildContext malformed_context {.scan_params = &malformed_scan_params}; |
| 1603 | + const auto status = reader.annotate_projected_column(TFileScanSlotInfo(), &malformed_context, |
| 1604 | + &malformed_column); |
| 1605 | + EXPECT_TRUE(status.is<ErrorCode::INVALID_ARGUMENT>()) << status; |
| 1606 | + EXPECT_NE(status.to_string().find("VARIANT initial-default for field 'payload' must be NULL"), |
| 1607 | + std::string::npos) |
| 1608 | + << status; |
| 1609 | +} |
| 1610 | + |
1508 | 1611 | TEST(IcebergV2ReaderTest, AnnotateBuildsComplexInitialDefaults) { |
1509 | 1612 | const auto required_int_type = std::make_shared<DataTypeInt32>(); |
1510 | 1613 | const auto optional_string_type = make_nullable(std::make_shared<DataTypeString>()); |
|
0 commit comments