Skip to content

Commit 9af6e56

Browse files
morningmanhubgeterclaude
committed
[fix](be) Parse complex Iceberg initial defaults at full precision
### What problem does this PR solve? Issue Number: None Related PR: #66538 Problem Summary: Iceberg stores the initial default of a struct, list or map field as single-value JSON. Both BE readers, FileScanner V1 (iceberg_default_value.h) and V2 (format_v2/table/iceberg_reader.cpp), parse that JSON with rapidjson's default Parse. Its fast path rounds a long decimal significand to a double and then scales it, so a double with 17 or more significant digits can be rounded twice and come back one ULP off: [0.18172760479972437302] decodes to 0x3fc742d9a3b296db instead of the correctly rounded 0x3fc742d9a3b296dc that Spark returns. Rows from data files written before the column existed then read a different value in Doris than in Spark. branch-4.1 #66538 parses this JSON with kParseFullPrecisionFlag in both readers. This change does the same and ports its two unit tests. ### Release note Fix: a DOUBLE inside the initial default of an Iceberg struct, list or map column could be read one ULP off; it now matches Spark exactly. ### Check List (For Author) - Test: Unit Test - IcebergReaderTest.materializes_complex_double_initial_default_at_full_precision and IcebergV2ReaderTest.AnnotateBuildsComplexDoubleDefaultAtFullPrecision fail without the fix (0x3fc742d9a3b296db) and pass with it; all 144 tests of IcebergReaderTest and IcebergV2ReaderTest pass - Behavior changed: Yes. Doubles inside complex initial defaults read back bit-identical to Spark instead of possibly one ULP off - Does this need documentation: No Co-authored-by: daidai <[email protected]> Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 73a987e commit 9af6e56

4 files changed

Lines changed: 82 additions & 2 deletions

File tree

‎be/src/format/table/iceberg_default_value.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -418,7 +418,8 @@ inline Status build_initial_default_field(const schema::external::TField& field,
418418
const auto primitive_type = value_type->get_primitive_type();
419419
if (is_complex_type(primitive_type)) {
420420
rapidjson::Document document;
421-
document.Parse(field.initial_default_value.data(), field.initial_default_value.size());
421+
document.Parse<rapidjson::kParseFullPrecisionFlag>(field.initial_default_value.data(),
422+
field.initial_default_value.size());
422423
if (document.HasParseError()) {
423424
return Status::InvalidArgument("Invalid Iceberg JSON initial default for field '{}'",
424425
field.name);

‎be/src/format_v2/table/iceberg_reader.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -616,7 +616,8 @@ static Status build_v2_initial_default_field(const format::ColumnDefinition& fie
616616
}
617617
if (is_complex_type(primitive_type)) {
618618
rapidjson::Document document;
619-
document.Parse(field.initial_default_value->data(), field.initial_default_value->size());
619+
document.Parse<rapidjson::kParseFullPrecisionFlag>(field.initial_default_value->data(),
620+
field.initial_default_value->size());
620621
if (document.HasParseError()) {
621622
return Status::InvalidArgument("Invalid Iceberg JSON initial default for field '{}'",
622623
field.name);

‎be/test/format/table/iceberg/iceberg_reader_test.cpp‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
#include <parquet/arrow/writer.h>
2929

3030
#include <array>
31+
#include <bit>
3132
#include <cmath>
3233
#include <filesystem>
3334
#include <fstream>
@@ -1403,6 +1404,44 @@ class IcebergReaderTest : public ::testing::Test {
14031404
{"pos", 1}};
14041405
};
14051406

1407+
TEST_F(IcebergReaderTest, materializes_complex_double_initial_default_at_full_precision) {
1408+
auto element = std::make_shared<schema::external::TField>();
1409+
element->__set_name("element");
1410+
element->__set_id(2);
1411+
element->__set_is_optional(false);
1412+
TColumnType element_thrift_type;
1413+
element_thrift_type.__set_type(TPrimitiveType::DOUBLE);
1414+
element->__set_type(element_thrift_type);
1415+
schema::external::TFieldPtr element_ptr;
1416+
element_ptr.__set_field_ptr(std::move(element));
1417+
1418+
schema::external::TField values;
1419+
values.__set_name("values");
1420+
values.__set_id(1);
1421+
values.__set_is_optional(true);
1422+
values.__set_initial_default_value("[0.18172760479972437302]");
1423+
TColumnType values_thrift_type;
1424+
values_thrift_type.__set_type(TPrimitiveType::ARRAY);
1425+
values.__set_type(values_thrift_type);
1426+
schema::external::TArrayField array_metadata;
1427+
array_metadata.__set_item_field(std::move(element_ptr));
1428+
values.nestedField.__set_array_field(std::move(array_metadata));
1429+
values.__isset.nestedField = true;
1430+
1431+
const auto double_type = std::make_shared<DataTypeFloat64>();
1432+
const auto values_type = make_nullable(std::make_shared<DataTypeArray>(double_type));
1433+
ColumnPtr default_column;
1434+
ASSERT_TRUE(iceberg::create_initial_default_column(values, values_type, &default_column).ok());
1435+
const auto& nullable = assert_cast<const ColumnNullable&>(*default_column);
1436+
ASSERT_FALSE(nullable.is_null_at(0));
1437+
const auto& array = assert_cast<const ColumnArray&>(nullable.get_nested_column());
1438+
const auto& nullable_doubles = assert_cast<const ColumnNullable&>(array.get_data());
1439+
ASSERT_FALSE(nullable_doubles.is_null_at(0));
1440+
const auto& doubles = assert_cast<const ColumnFloat64&>(nullable_doubles.get_nested_column());
1441+
ASSERT_EQ(doubles.size(), 1);
1442+
EXPECT_EQ(std::bit_cast<uint64_t>(doubles.get_data()[0]), 0x3fc742d9a3b296dcULL);
1443+
}
1444+
14061445
TEST_F(IcebergReaderTest, detects_fully_dictionary_encoded_parquet_column) {
14071446
tparquet::ColumnMetaData column_metadata;
14081447
column_metadata.type = tparquet::Type::BYTE_ARRAY;

‎be/test/format_v2/table/iceberg_reader_test.cpp‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626

2727
#include <algorithm>
2828
#include <array>
29+
#include <bit>
2930
#include <chrono>
3031
#include <cmath>
3132
#include <cstring>
@@ -2130,6 +2131,44 @@ TEST(IcebergV2ReaderTest, AnnotateBuildsComplexInitialDefaults) {
21302131
EXPECT_EQ(values[0].get<TYPE_INT>(), 9);
21312132
}
21322133

2134+
TEST(IcebergV2ReaderTest, AnnotateBuildsComplexDoubleDefaultAtFullPrecision) {
2135+
const auto double_type = std::make_shared<DataTypeFloat64>();
2136+
const auto list_type = make_nullable(std::make_shared<DataTypeArray>(double_type));
2137+
auto list_element =
2138+
external_schema_field("element", 2, {}, std::nullopt,
2139+
external_primitive_type(TPrimitiveType::DOUBLE), false, false);
2140+
auto list_field = external_schema_field("values", 1, {}, "[0.18172760479972437302]",
2141+
std::nullopt, false, true);
2142+
schema::external::TArrayField array_metadata;
2143+
array_metadata.__set_item_field(std::move(list_element));
2144+
list_field.field_ptr->nestedField.__set_array_field(std::move(array_metadata));
2145+
list_field.field_ptr->__isset.nestedField = true;
2146+
TFileScanRangeParams scan_params;
2147+
scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2);
2148+
scan_params.__set_current_schema_id(100);
2149+
scan_params.__set_history_schema_info({external_schema(100, {std::move(list_field)})});
2150+
2151+
ColumnDefinition column;
2152+
column.name = "values";
2153+
column.type = list_type;
2154+
ProjectedColumnBuildContext context {.scan_params = &scan_params};
2155+
doris::format::iceberg::IcebergTableReader reader;
2156+
const auto status = reader.annotate_projected_column(TFileScanSlotInfo(), &context, &column);
2157+
ASSERT_TRUE(status.ok()) << status;
2158+
ASSERT_NE(column.default_expr, nullptr);
2159+
const auto* literal = dynamic_cast<const VLiteral*>(column.default_expr->root().get());
2160+
ASSERT_NE(literal, nullptr);
2161+
const auto literal_column = literal->get_column_ptr()->convert_to_full_column_if_const();
2162+
const auto& nullable = assert_cast<const ColumnNullable&>(*literal_column);
2163+
ASSERT_FALSE(nullable.is_null_at(0));
2164+
const auto& array = assert_cast<const ColumnArray&>(nullable.get_nested_column());
2165+
const auto& nullable_doubles = assert_cast<const ColumnNullable&>(array.get_data());
2166+
ASSERT_FALSE(nullable_doubles.is_null_at(0));
2167+
const auto& doubles = assert_cast<const ColumnFloat64&>(nullable_doubles.get_nested_column());
2168+
ASSERT_EQ(doubles.size(), 1);
2169+
EXPECT_EQ(std::bit_cast<uint64_t>(doubles.get_data()[0]), 0x3fc742d9a3b296dcULL);
2170+
}
2171+
21332172
TEST(IcebergV2ReaderTest, ComplexInitialDefaultPrefersExactChildNameOverAlias) {
21342173
const auto int_type = std::make_shared<DataTypeInt32>();
21352174
const auto struct_type =

0 commit comments

Comments
 (0)