Skip to content

Commit d2b6763

Browse files
committed
[feature](iceberg) Enforce NULL defaults for VARIANT
1 parent 2c3ae38 commit d2b6763

9 files changed

Lines changed: 368 additions & 0 deletions

File tree

‎be/src/format_v2/table/iceberg_reader.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,10 @@ static Status build_v2_json_scalar_default(const format::ColumnDefinition& field
311311
std::deque<std::string>* binary_storage,
312312
const cctz::time_zone* timezone, Field* result) {
313313
const auto primitive_type = value_type->get_primitive_type();
314+
if (primitive_type == TYPE_VARIANT) {
315+
return Status::InvalidArgument(
316+
"Iceberg VARIANT initial-default for field '{}' must be NULL", field.name);
317+
}
314318
std::string serialized_value = iceberg_json_scalar_text(json_value);
315319
const bool binary_like =
316320
field.initial_default_value_is_base64 || primitive_type == TYPE_VARBINARY;
@@ -395,6 +399,10 @@ static Status build_v2_initial_default_field(const format::ColumnDefinition& fie
395399

396400
const auto value_type = remove_nullable(data_type);
397401
const auto primitive_type = value_type->get_primitive_type();
402+
if (primitive_type == TYPE_VARIANT) {
403+
return Status::InvalidArgument(
404+
"Iceberg VARIANT initial-default for field '{}' must be NULL", field.name);
405+
}
398406
if (is_complex_type(primitive_type)) {
399407
rapidjson::Document document;
400408
document.Parse<rapidjson::kParseFullPrecisionFlag>(field.initial_default_value->data(),

‎be/src/format_v2/table_reader.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -511,6 +511,11 @@ DataTypePtr restore_current_primitive_type(const schema::external::TField& field
511511
}
512512
DORIS_CHECK(fallback_type != nullptr);
513513
const auto primitive_type = thrift_to_type(field.type.type);
514+
if (primitive_type == TYPE_VARIANT) {
515+
// TColumnType predates the execution-only variant_is_v2 marker. Reconstructing from its
516+
// primitive enum would silently replace an Iceberg compute-V2 carrier with legacy VARIANT.
517+
return fallback_type;
518+
}
514519
if (is_complex_type(primitive_type)) {
515520
return fallback_type;
516521
}

‎be/test/format_v2/table/iceberg_reader_test.cpp‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@
6060
#include "core/data_type/data_type_struct.h"
6161
#include "core/data_type/data_type_timestamptz.h"
6262
#include "core/data_type/data_type_varbinary.h"
63+
#include "core/data_type/data_type_variant_v2.h"
6364
#include "exec/common/endian.h"
6465
#include "exec/scan/access_path_parser.h"
6566
#include "exprs/runtime_filter_expr.h"
@@ -1505,6 +1506,108 @@ TEST(IcebergV2ReaderTest, AnnotateConvertsTimestamptzDefaultToSessionTimezone) {
15051506
"2025-01-18 09:02:03.654321");
15061507
}
15071508

1509+
TEST(IcebergV2ReaderTest, AnnotateVariantInitialDefaultMustBeNull) {
1510+
const auto variant_type = make_nullable(std::make_shared<DataTypeVariantV2>());
1511+
1512+
auto optional_variant =
1513+
external_schema_field("payload", 1, {}, std::nullopt,
1514+
external_primitive_type(TPrimitiveType::VARIANT), false, true);
1515+
TFileScanRangeParams valid_scan_params;
1516+
valid_scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2);
1517+
valid_scan_params.__set_current_schema_id(100);
1518+
valid_scan_params.__set_history_schema_info(
1519+
{external_schema(100, {std::move(optional_variant)})});
1520+
1521+
ColumnDefinition valid_column;
1522+
valid_column.name = "payload";
1523+
valid_column.type = variant_type;
1524+
ProjectedColumnBuildContext valid_context {.scan_params = &valid_scan_params};
1525+
doris::format::iceberg::IcebergTableReader reader;
1526+
ASSERT_TRUE(reader.annotate_projected_column(TFileScanSlotInfo(), &valid_context, &valid_column)
1527+
.ok());
1528+
ASSERT_TRUE(valid_column.is_optional.has_value());
1529+
EXPECT_TRUE(*valid_column.is_optional);
1530+
EXPECT_FALSE(valid_column.initial_default_value.has_value());
1531+
EXPECT_EQ(valid_column.default_expr, nullptr);
1532+
ASSERT_TRUE(valid_context.schema_column.has_value());
1533+
EXPECT_TRUE(valid_context.schema_column->type->equals(*variant_type));
1534+
1535+
auto malformed_variant =
1536+
external_schema_field("payload", 1, {}, "{\"source\":\"malformed\"}",
1537+
external_primitive_type(TPrimitiveType::VARIANT), false, true);
1538+
TFileScanRangeParams malformed_scan_params;
1539+
malformed_scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2);
1540+
malformed_scan_params.__set_current_schema_id(101);
1541+
malformed_scan_params.__set_history_schema_info(
1542+
{external_schema(101, {std::move(malformed_variant)})});
1543+
1544+
ColumnDefinition malformed_column;
1545+
malformed_column.name = "payload";
1546+
malformed_column.type = variant_type;
1547+
ProjectedColumnBuildContext malformed_context {.scan_params = &malformed_scan_params};
1548+
const auto status = reader.annotate_projected_column(TFileScanSlotInfo(), &malformed_context,
1549+
&malformed_column);
1550+
EXPECT_TRUE(status.is<ErrorCode::INVALID_ARGUMENT>()) << status;
1551+
EXPECT_NE(status.to_string().find("VARIANT initial-default for field 'payload' must be NULL"),
1552+
std::string::npos)
1553+
<< status;
1554+
}
1555+
1556+
TEST(IcebergV2ReaderTest, StructInitialDefaultKeepsVariantChildNull) {
1557+
const auto variant_type = make_nullable(std::make_shared<DataTypeVariantV2>());
1558+
const auto struct_type = make_nullable(
1559+
std::make_shared<DataTypeStruct>(DataTypes {variant_type}, Strings {"payload"}));
1560+
1561+
auto optional_variant =
1562+
external_schema_field("payload", 2, {}, std::nullopt,
1563+
external_primitive_type(TPrimitiveType::VARIANT), false, true);
1564+
auto valid_struct =
1565+
external_struct_schema_field("event", 1, {std::move(optional_variant)}, true, "{}");
1566+
TFileScanRangeParams valid_scan_params;
1567+
valid_scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2);
1568+
valid_scan_params.__set_current_schema_id(100);
1569+
valid_scan_params.__set_history_schema_info({external_schema(100, {std::move(valid_struct)})});
1570+
1571+
ColumnDefinition valid_column;
1572+
valid_column.name = "event";
1573+
valid_column.type = struct_type;
1574+
ProjectedColumnBuildContext valid_context {.scan_params = &valid_scan_params};
1575+
doris::format::iceberg::IcebergTableReader reader;
1576+
ASSERT_TRUE(reader.annotate_projected_column(TFileScanSlotInfo(), &valid_context, &valid_column)
1577+
.ok());
1578+
ASSERT_NE(valid_column.default_expr, nullptr);
1579+
const auto* literal = dynamic_cast<const VLiteral*>(valid_column.default_expr->root().get());
1580+
ASSERT_NE(literal, nullptr);
1581+
EXPECT_TRUE(literal->get_data_type()->equals(*struct_type));
1582+
Field value;
1583+
literal->get_column_ptr()->get(0, value);
1584+
const auto& struct_fields = value.get<TYPE_STRUCT>();
1585+
ASSERT_EQ(struct_fields.size(), 1);
1586+
EXPECT_TRUE(struct_fields[0].is_null());
1587+
1588+
auto malformed_variant =
1589+
external_schema_field("payload", 2, {}, std::nullopt,
1590+
external_primitive_type(TPrimitiveType::VARIANT), false, true);
1591+
auto malformed_struct = external_struct_schema_field(
1592+
"event", 1, {std::move(malformed_variant)}, true, "{\"2\":{\"source\":\"malformed\"}}");
1593+
TFileScanRangeParams malformed_scan_params;
1594+
malformed_scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2);
1595+
malformed_scan_params.__set_current_schema_id(101);
1596+
malformed_scan_params.__set_history_schema_info(
1597+
{external_schema(101, {std::move(malformed_struct)})});
1598+
1599+
ColumnDefinition malformed_column;
1600+
malformed_column.name = "event";
1601+
malformed_column.type = struct_type;
1602+
ProjectedColumnBuildContext malformed_context {.scan_params = &malformed_scan_params};
1603+
const auto status = reader.annotate_projected_column(TFileScanSlotInfo(), &malformed_context,
1604+
&malformed_column);
1605+
EXPECT_TRUE(status.is<ErrorCode::INVALID_ARGUMENT>()) << status;
1606+
EXPECT_NE(status.to_string().find("VARIANT initial-default for field 'payload' must be NULL"),
1607+
std::string::npos)
1608+
<< status;
1609+
}
1610+
15081611
TEST(IcebergV2ReaderTest, AnnotateBuildsComplexInitialDefaults) {
15091612
const auto required_int_type = std::make_shared<DataTypeInt32>();
15101613
const auto optional_string_type = make_nullable(std::make_shared<DataTypeString>());

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1224,6 +1224,8 @@ public static Literal<?> parseIcebergLiteral(String value, org.apache.iceberg.ty
12241224
} catch (NumberFormatException e) {
12251225
throw new IllegalArgumentException("Invalid Decimal string: " + value, e);
12261226
}
1227+
case VARIANT:
1228+
throw new IllegalArgumentException("Iceberg VARIANT default values must be NULL");
12271229
default:
12281230
throw new IllegalArgumentException("Cannot parse unknown type: " + type);
12291231
}
@@ -1427,6 +1429,9 @@ public static Column parseField(Types.NestedField field, boolean enableMappingVa
14271429

14281430
private static String serializeInitialDefault(org.apache.iceberg.types.Type type, Object value,
14291431
boolean enableMappingTimestampTz) {
1432+
if (type.typeId() == TypeID.VARIANT) {
1433+
throw new IllegalArgumentException("Iceberg VARIANT initial-default must be NULL");
1434+
}
14301435
if (type.isNestedType()) {
14311436
// Keep Iceberg's type-directed JSON representation for struct/list/map values. In
14321437
// particular, struct members are keyed by field id and an empty object is the V3

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergWriteSchemaContext.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -641,6 +641,8 @@ static Expression toDorisExpression(Type icebergType, Object value, DataType tar
641641
case STRUCT:
642642
return structExpression((Types.StructType) icebergType, value, targetType,
643643
enableMappingVarbinary, enableMappingTimestampTz);
644+
case VARIANT:
645+
throw new AnalysisException("Iceberg VARIANT write-default must be NULL");
644646
default:
645647
throw new AnalysisException("Unsupported Iceberg write-default type: " + icebergType);
646648
}

‎fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -394,6 +394,27 @@ public void testIcebergVariantUsesComputeV2Representation() {
394394
Assert.assertEquals(Types.VariantType.get(), IcebergUtils.dorisTypeToIcebergType(type));
395395
}
396396

397+
@Test
398+
public void testIcebergVariantDefaultsMustBeNull() {
399+
Types.VariantType variantType = Types.VariantType.get();
400+
Schema schema = new Schema(Types.NestedField.optional(1, "payload", variantType));
401+
402+
Assert.assertNull(IcebergUtils.parseIcebergLiteral(null, variantType));
403+
Assert.assertTrue(IcebergUtils.getSerializedInitialDefaults(schema, false).isEmpty());
404+
405+
IllegalArgumentException ddlException = Assert.assertThrows(IllegalArgumentException.class,
406+
() -> IcebergUtils.parseIcebergLiteral("{\"source\":\"ddl\"}", variantType));
407+
Assert.assertTrue(ddlException.getMessage().contains("VARIANT default values must be NULL"));
408+
409+
Types.NestedField malformedField = Mockito.mock(Types.NestedField.class);
410+
Mockito.when(malformedField.fieldId()).thenReturn(1);
411+
Mockito.when(malformedField.type()).thenReturn(variantType);
412+
Mockito.when(malformedField.initialDefault()).thenReturn("non-null-variant");
413+
IllegalArgumentException readException = Assert.assertThrows(IllegalArgumentException.class,
414+
() -> IcebergUtils.getSerializedInitialDefault(malformedField, false));
415+
Assert.assertTrue(readException.getMessage().contains("VARIANT initial-default must be NULL"));
416+
}
417+
397418
@Test
398419
public void testIcebergVariantWriteCapabilityMatrix() {
399420
Type variant = IcebergUtils.icebergTypeToDorisType(Types.VariantType.get(), false, false);

‎fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergWriteSchemaContextTest.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,48 @@ public void testLegacyBinaryDefaultsDecodeRawBytesOnBackend() {
217217
assertUnhexBytes(uuid, "123E4567E89B12D3A456426614174000");
218218
}
219219

220+
@Test
221+
public void testVariantWriteDefaultMustBeNull() {
222+
Types.VariantType variantType = Types.VariantType.get();
223+
Schema schema = new Schema(18, Arrays.asList(
224+
Types.NestedField.optional(1, "optional_payload", variantType),
225+
Types.NestedField.required(2, "required_payload", variantType)));
226+
IcebergWriteSchemaContext context = IcebergWriteSchemaContext.forSchema(
227+
schema, 3, true, true);
228+
Map<String, Column> columns = context.getColumns().stream()
229+
.collect(Collectors.toMap(Column::getName, column -> column));
230+
231+
Expression optionalDefault = context.resolveWriteDefault(columns.get("optional_payload"));
232+
Assertions.assertTrue(optionalDefault instanceof NullLiteral);
233+
Assertions.assertEquals(DataType.fromCatalogType(columns.get("optional_payload").getType()),
234+
optionalDefault.getDataType());
235+
Assertions.assertThrows(AnalysisException.class,
236+
() -> context.resolveWriteDefault(columns.get("required_payload")));
237+
238+
AnalysisException nonNullException = Assertions.assertThrows(AnalysisException.class,
239+
() -> IcebergWriteSchemaContext.toDorisExpression(
240+
variantType, new Object(), optionalDefault.getDataType(), true, true));
241+
Assertions.assertTrue(nonNullException.getMessage().contains(
242+
"VARIANT write-default must be NULL"));
243+
}
244+
245+
@Test
246+
public void testStructWriteDefaultKeepsVariantChildNull() {
247+
Types.StructType structType = Types.StructType.of(
248+
Types.NestedField.optional(101, "payload", Types.VariantType.get()));
249+
DataType targetType = DataType.fromCatalogType(IcebergUtils.icebergTypeToDorisType(
250+
structType, true, true));
251+
252+
Expression expression = IcebergWriteSchemaContext.toDorisExpression(
253+
structType, new ArrayStructLike((Object) null), targetType, true, true);
254+
255+
Assertions.assertTrue(expression instanceof StructLiteral);
256+
List<?> fields = ((StructLiteral) expression).getValue();
257+
Assertions.assertEquals(1, fields.size());
258+
Assertions.assertTrue(fields.get(0) instanceof NullLiteral);
259+
Assertions.assertEquals(targetType, expression.getDataType());
260+
}
261+
220262
@Test
221263
public void testWriteDefaultResolutionDoesNotFallBackToReusedName() {
222264
Schema pinnedSchema = new Schema(18, ImmutableList.of(defaultField(
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !variant_default_historical_row --
3+
1 true \N
4+
5+
-- !variant_default_rows --
6+
1 true \N
7+
2 true \N
8+
3 true \N
9+
4 true \N
10+
5 true \N
11+
6 true \N
12+
7 false null
13+
8 true \N
14+
15+
-- !variant_default_counts --
16+
8 1 7
17+
18+
-- !variant_explicit_default_null_column --
19+
8 0 8
20+
21+
-- !variant_orc_historical_null --
22+
10 true \N

0 commit comments

Comments
 (0)