Skip to content

Commit a82564c

Browse files
authored
[fix](iceberg) Fix MVCC and nested schema evolution edge cases (#66345)
### What problem does this PR solve? This PR fixes correctness and memory-usage edge cases in Iceberg MVCC writes and nested schema evolution: - An explicitly empty read could be reinterpreted as the latest snapshot after a concurrent first append. A MERGE could then miss the concurrent row and silently insert a duplicate. - Write-schema validation used the INSERT column subset instead of the complete bind-time target schema, rejecting valid partial and static-partition INSERTs while weakening ordinal drift checks. - Iceberg v3 row-lineage metadata columns could be mistaken for user-schema drift. - Parent null-mask handling for nested ARRAY/MAP values could scan descendant entry maps and allocate entry-sized scratch even when the inherited mask was absent or all-clear. - Strict cast failure semantics were incorrectly used to mark a struct field physically non-nullable even though BE narrowing casts still return `ColumnNullable`. - Nested DESCRIBE comments were not emitted as round-trippable SQL literals. The fix preserves the explicit empty-snapshot marker through scan planning and keeps RowDelta conflict validation anchored at table creation. In the concurrent empty-table MERGE/INSERT window, MERGE keeps reading the original empty view and its commit conflicts with the concurrent first append instead of producing duplicate data. Write planning now carries an immutable complete target-schema snapshot separately from the write subset, validates column order, type, and Iceberg field id, and excludes engine-generated row-lineage metadata from the user-schema comparison. Nested collection alignment gates descendant scans on an inherited mask that can actually hide a row, and struct constructors preserve the physical nullability of cast results. ### Release note Fix Iceberg empty-snapshot concurrency, partial-write schema validation, nested collection memory usage, cast-result nullability, and nested DESCRIBE comment rendering. ### Check List (For Author) - Test - [x] Regression tests - [x] Unit tests - Behavior changed: - [x] Yes. Correctness and memory-usage fixes for the affected edge cases. - Does this need documentation? - [x] No. ### Tests - Iceberg write-plan tests: 50 passed - Iceberg concurrent empty-snapshot barrier test: passed - BE `TableReaderTest`: 95 passed - FE connector sink, binding, executor, and struct tests: 38 passed - FE Checkstyle: 0 violations - Clang Formatter 16: passed - `git diff --check`: passed
1 parent 38fb05e commit a82564c

78 files changed

Lines changed: 4149 additions & 426 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/format_v2/table_reader.h‎

Lines changed: 220 additions & 21 deletions
Large diffs are not rendered by default.

‎be/test/format_v2/table_reader_test.cpp‎

Lines changed: 225 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1064,7 +1064,9 @@ class TableReaderCastTestHelper final : public TableReader {
10641064
using TableReader::_materialize_mapping_column;
10651065
using TableReader::_materialize_present_child_mapping_column;
10661066
using TableReader::_materialize_struct_mapping_column;
1067-
using TableReader::_project_collection_parent_null_map;
1067+
using TableReader::_project_collection_parent_null_map_for_hidden_entries;
1068+
using TableReader::_requires_collection_parent_null_map;
1069+
using TableReader::_requires_parent_null_map_for_alignment;
10681070
};
10691071

10701072
TEST(TableReaderTest, TruncateCharOrVarcharPredicateOnlyAppliesToParquetStringWidthMismatch) {
@@ -3149,6 +3151,76 @@ TEST(TableReaderTest, ScalarProjectionRejectsNullInRequiredTableColumn) {
31493151
std::string::npos);
31503152
}
31513153

3154+
TEST(TableReaderTest, CollectionParentMaskSkipsLargeArrayWhenNearerMaskCoversVisibleNull) {
3155+
constexpr size_t visible_entries = 500000;
3156+
const size_t entries = visible_entries + 1;
3157+
const auto int_type = std::make_shared<DataTypeInt32>();
3158+
const auto struct_type =
3159+
std::make_shared<DataTypeStruct>(DataTypes {int_type}, Strings {"value"});
3160+
const auto nullable_struct_type = make_nullable(struct_type);
3161+
3162+
auto values = ColumnInt32::create(entries, 0);
3163+
auto value_null_map = ColumnUInt8::create(entries, 0);
3164+
value_null_map->get_data().back() = 1;
3165+
MutableColumns struct_children;
3166+
struct_children.push_back(ColumnNullable::create(std::move(values), std::move(value_null_map)));
3167+
auto element_null_map = ColumnUInt8::create(entries, 0);
3168+
element_null_map->get_data().back() = 1;
3169+
ColumnPtr nullable_elements = ColumnNullable::create(
3170+
ColumnStruct::create(std::move(struct_children)), std::move(element_null_map));
3171+
NullMap parent_null_map {1, 0};
3172+
ColumnArray::Offsets64 offsets {1, entries};
3173+
3174+
EXPECT_FALSE(TableReaderCastTestHelper::_requires_collection_parent_null_map(
3175+
&parent_null_map, nullable_elements, nullable_struct_type, 2, offsets));
3176+
}
3177+
3178+
TEST(TableReaderTest, CollectionParentMaskSkipsLargeMapWhenNearerMaskCoversVisibleNull) {
3179+
constexpr size_t visible_entries = 500000;
3180+
const size_t entries = visible_entries + 1;
3181+
const auto int_type = std::make_shared<DataTypeInt32>();
3182+
const auto struct_type =
3183+
std::make_shared<DataTypeStruct>(DataTypes {int_type}, Strings {"value"});
3184+
const auto nullable_struct_type = make_nullable(struct_type);
3185+
3186+
auto values = ColumnInt32::create(entries, 0);
3187+
auto value_null_map = ColumnUInt8::create(entries, 0);
3188+
value_null_map->get_data().back() = 1;
3189+
MutableColumns struct_children;
3190+
struct_children.push_back(ColumnNullable::create(std::move(values), std::move(value_null_map)));
3191+
auto value_struct_null_map = ColumnUInt8::create(entries, 0);
3192+
value_struct_null_map->get_data().back() = 1;
3193+
ColumnPtr nullable_value_structs = ColumnNullable::create(
3194+
ColumnStruct::create(std::move(struct_children)), std::move(value_struct_null_map));
3195+
NullMap parent_null_map {1, 0};
3196+
ColumnArray::Offsets64 offsets {1, entries};
3197+
3198+
EXPECT_FALSE(TableReaderCastTestHelper::_requires_collection_parent_null_map(
3199+
&parent_null_map, nullable_value_structs, nullable_struct_type, 2, offsets));
3200+
}
3201+
3202+
TEST(TableReaderTest, CollectionParentMaskKeepsRequiredNullInHiddenEntry) {
3203+
const auto int_type = std::make_shared<DataTypeInt32>();
3204+
const auto struct_type =
3205+
std::make_shared<DataTypeStruct>(DataTypes {int_type}, Strings {"value"});
3206+
const auto nullable_struct_type = make_nullable(struct_type);
3207+
3208+
auto values = ColumnInt32::create(2, 0);
3209+
auto value_null_map = ColumnUInt8::create();
3210+
value_null_map->get_data().assign({1, 1});
3211+
MutableColumns struct_children;
3212+
struct_children.push_back(ColumnNullable::create(std::move(values), std::move(value_null_map)));
3213+
auto element_null_map = ColumnUInt8::create();
3214+
element_null_map->get_data().assign({0, 1});
3215+
ColumnPtr nullable_elements = ColumnNullable::create(
3216+
ColumnStruct::create(std::move(struct_children)), std::move(element_null_map));
3217+
NullMap parent_null_map {1, 0};
3218+
ColumnArray::Offsets64 offsets {1, 2};
3219+
3220+
EXPECT_TRUE(TableReaderCastTestHelper::_requires_collection_parent_null_map(
3221+
&parent_null_map, nullable_elements, nullable_struct_type, 2, offsets));
3222+
}
3223+
31523224
TEST(TableReaderTest, ReopenSplitAfterClose) {
31533225
const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_test";
31543226
std::filesystem::remove_all(test_dir);
@@ -4924,6 +4996,112 @@ TEST(TableReaderTest, CreateScanRequestDeduplicatesSharedPredicateColumns) {
49244996
}
49254997
}
49264998

4999+
TEST(TableReaderTest, TrivialArrayChildProjectsNullableStructParentMask) {
5000+
const auto int_type = std::make_shared<DataTypeInt32>();
5001+
const auto element_struct_type =
5002+
std::make_shared<DataTypeStruct>(DataTypes {int_type}, Strings {"value"});
5003+
const auto nullable_element_struct_type = make_nullable(element_struct_type);
5004+
const auto array_type = std::make_shared<DataTypeArray>(nullable_element_struct_type);
5005+
const auto table_struct_type = make_nullable(std::make_shared<DataTypeStruct>(
5006+
DataTypes {array_type, int_type}, Strings {"items", "added"}));
5007+
const auto file_struct_type = make_nullable(
5008+
std::make_shared<DataTypeStruct>(DataTypes {array_type}, Strings {"items"}));
5009+
5010+
auto table_items = make_table_column(0, "items", array_type);
5011+
table_items.type = array_type;
5012+
auto table_element = make_table_column(0, "element", element_struct_type);
5013+
table_element.type = element_struct_type;
5014+
auto table_value = make_table_column(0, "value", int_type);
5015+
table_value.type = int_type;
5016+
table_element.children = {table_value};
5017+
table_items.children = {table_element};
5018+
auto table_added = make_table_column(1, "added", int_type);
5019+
table_added.type = int_type;
5020+
auto table_struct = make_table_column(0, "payload", table_struct_type);
5021+
table_struct.type = table_struct_type;
5022+
table_struct.children = {table_items, table_added};
5023+
5024+
auto file_items = make_file_column(0, "items", array_type);
5025+
file_items.type = array_type;
5026+
auto file_element = make_file_column(0, "element", element_struct_type);
5027+
file_element.type = element_struct_type;
5028+
auto file_value = make_file_column(0, "value", int_type);
5029+
file_value.type = int_type;
5030+
file_element.children = {file_value};
5031+
file_items.children = {file_element};
5032+
auto file_struct = make_file_column(0, "payload", file_struct_type);
5033+
file_struct.type = file_struct_type;
5034+
file_struct.children = {file_items};
5035+
5036+
TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME});
5037+
ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok());
5038+
ASSERT_EQ(mapper.mappings().size(), 1);
5039+
ASSERT_FALSE(mapper.mappings()[0].is_trivial);
5040+
ASSERT_TRUE(mapper.mappings()[0].child_mappings[0].is_trivial);
5041+
5042+
auto values = ColumnInt32::create();
5043+
values->get_data().assign({0, 7});
5044+
auto value_null_map = ColumnUInt8::create();
5045+
value_null_map->get_data().assign({1, 0});
5046+
MutableColumns element_children;
5047+
element_children.push_back(
5048+
ColumnNullable::create(std::move(values), std::move(value_null_map)));
5049+
auto element_null_map = ColumnUInt8::create(2, 0);
5050+
auto array_values = ColumnNullable::create(ColumnStruct::create(std::move(element_children)),
5051+
std::move(element_null_map));
5052+
auto offsets = ColumnArray::ColumnOffsets::create();
5053+
offsets->get_data().assign({1, 2});
5054+
MutableColumns struct_children;
5055+
struct_children.push_back(ColumnArray::create(std::move(array_values), std::move(offsets)));
5056+
auto parent_null_map = ColumnUInt8::create();
5057+
parent_null_map->get_data().assign({1, 0});
5058+
ColumnPtr file_data = ColumnNullable::create(ColumnStruct::create(std::move(struct_children)),
5059+
std::move(parent_null_map));
5060+
5061+
TableReaderCastTestHelper reader;
5062+
ColumnPtr result;
5063+
const auto status =
5064+
reader._materialize_struct_mapping_column(mapper.mappings()[0], file_data, 2, &result);
5065+
ASSERT_TRUE(status.ok()) << status.to_string();
5066+
const auto& result_struct = assert_cast<const ColumnStruct&>(
5067+
assert_cast<const ColumnNullable&>(*result).get_nested_column());
5068+
const auto& result_array = assert_cast<const ColumnArray&>(result_struct.get_column(0));
5069+
const auto& result_elements = assert_cast<const ColumnNullable&>(result_array.get_data());
5070+
const auto& result_element_struct =
5071+
assert_cast<const ColumnStruct&>(result_elements.get_nested_column());
5072+
ASSERT_FALSE(result_element_struct.get_column(0).is_nullable());
5073+
EXPECT_EQ(assert_cast<const ColumnInt32&>(result_element_struct.get_column(0)).get_element(1),
5074+
7);
5075+
}
5076+
5077+
TEST(TableReaderTest, ParentMaskProjectionOnlyWhenRequiredDescendantCanConsumeIt) {
5078+
const auto int_type = std::make_shared<DataTypeInt32>();
5079+
auto mutable_values = ColumnInt32::create();
5080+
mutable_values->get_data().assign({1, 2});
5081+
ColumnPtr values = std::move(mutable_values);
5082+
EXPECT_FALSE(
5083+
TableReaderCastTestHelper::_requires_parent_null_map_for_alignment(values, int_type));
5084+
5085+
ColumnPtr nullable_values = ColumnNullable::create(values->clone(), ColumnUInt8::create(2, 0));
5086+
EXPECT_FALSE(TableReaderCastTestHelper::_requires_parent_null_map_for_alignment(nullable_values,
5087+
int_type));
5088+
auto null_map = ColumnUInt8::create(2, 0);
5089+
null_map->get_data()[0] = 1;
5090+
ColumnPtr nullable_values_with_null =
5091+
ColumnNullable::create(values->clone(), std::move(null_map));
5092+
EXPECT_TRUE(TableReaderCastTestHelper::_requires_parent_null_map_for_alignment(
5093+
nullable_values_with_null, int_type));
5094+
5095+
NullMap all_clear_parent_mask(2, 0);
5096+
EXPECT_FALSE(TableReaderCastTestHelper::_requires_collection_parent_null_map(
5097+
&all_clear_parent_mask, nullable_values_with_null, int_type));
5098+
NullMap hidden_parent_mask {1, 0};
5099+
EXPECT_TRUE(TableReaderCastTestHelper::_requires_collection_parent_null_map(
5100+
&hidden_parent_mask, nullable_values_with_null, int_type));
5101+
EXPECT_FALSE(TableReaderCastTestHelper::_requires_collection_parent_null_map(
5102+
nullptr, nullable_values_with_null, int_type));
5103+
}
5104+
49275105
TEST(TableReaderTest, CreateScanRequestPromotesProjectedColumnToPredicateColumn) {
49285106
const auto int_type = std::make_shared<DataTypeInt32>();
49295107
const std::vector<ColumnDefinition> projected_columns = {
@@ -6452,9 +6630,52 @@ TEST(TableReaderTest, CollectionParentMaskFastPathSkipsEntryScratchForClearMasks
64526630
ColumnArray::Offsets64 offsets {500000, 1000000};
64536631
NullMap projected_null_map;
64546632

6455-
const auto* result = TableReaderCastTestHelper::_project_collection_parent_null_map(
6456-
&container_null_map, &ancestor_null_map, 2, offsets, offsets.back(),
6457-
&projected_null_map);
6633+
const auto* result =
6634+
TableReaderCastTestHelper::_project_collection_parent_null_map_for_hidden_entries(
6635+
&container_null_map, &ancestor_null_map, 2, offsets, offsets.back(),
6636+
&projected_null_map);
6637+
6638+
EXPECT_EQ(nullptr, result);
6639+
EXPECT_TRUE(projected_null_map.empty());
6640+
}
6641+
6642+
TEST(TableReaderTest, CollectionParentMaskSkipsLargeArrayWhenOnlyEmptyRowIsHidden) {
6643+
constexpr size_t entries = 500000;
6644+
auto values = ColumnInt32::create(entries, 0);
6645+
auto value_null_map = ColumnUInt8::create(entries, 0);
6646+
value_null_map->get_data().back() = 1;
6647+
ColumnPtr nullable_values =
6648+
ColumnNullable::create(std::move(values), std::move(value_null_map));
6649+
NullMap parent_null_map {1, 0};
6650+
ColumnArray::Offsets64 offsets {0, entries};
6651+
NullMap projected_null_map;
6652+
6653+
EXPECT_FALSE(TableReaderCastTestHelper::_requires_collection_parent_null_map(
6654+
&parent_null_map, nullable_values, std::make_shared<DataTypeInt32>(), 2, offsets));
6655+
const auto* result =
6656+
TableReaderCastTestHelper::_project_collection_parent_null_map_for_hidden_entries(
6657+
nullptr, &parent_null_map, 2, offsets, entries, &projected_null_map);
6658+
6659+
EXPECT_EQ(nullptr, result);
6660+
EXPECT_TRUE(projected_null_map.empty());
6661+
}
6662+
6663+
TEST(TableReaderTest, CollectionParentMaskSkipsLargeMapWhenOnlyEmptyRowIsHidden) {
6664+
constexpr size_t entries = 500000;
6665+
auto values = ColumnInt32::create(entries, 0);
6666+
auto value_null_map = ColumnUInt8::create(entries, 0);
6667+
value_null_map->get_data().back() = 1;
6668+
ColumnPtr nullable_values =
6669+
ColumnNullable::create(std::move(values), std::move(value_null_map));
6670+
NullMap parent_null_map {1, 0};
6671+
ColumnArray::Offsets64 offsets {0, entries};
6672+
NullMap projected_null_map;
6673+
6674+
EXPECT_FALSE(TableReaderCastTestHelper::_requires_collection_parent_null_map(
6675+
&parent_null_map, nullable_values, std::make_shared<DataTypeInt32>(), 2, offsets));
6676+
const auto* result =
6677+
TableReaderCastTestHelper::_project_collection_parent_null_map_for_hidden_entries(
6678+
nullptr, &parent_null_map, 2, offsets, entries, &projected_null_map);
64586679

64596680
EXPECT_EQ(nullptr, result);
64606681
EXPECT_TRUE(projected_null_map.empty());

‎fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -656,6 +656,10 @@ public String getDefaultValueSql() {
656656
}
657657
}
658658

659+
public String getConnectorDefaultValueSql() {
660+
return connectorDefaultValueSql;
661+
}
662+
659663
public void setConnectorDefaultValueSql(String connectorDefaultValueSql) {
660664
this.connectorDefaultValueSql = connectorDefaultValueSql;
661665
}

‎fe/fe-common/src/main/java/org/apache/doris/common/util/SqlUtils.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,12 @@ public static String escapeQuota(String str) {
5454
return str.replaceAll("\"", "\\\\\"");
5555
}
5656

57+
/** Quote a value as a SQL string literal under the requested backslash-escape mode. */
58+
public static String quoteStringLiteral(String value, boolean noBackslashEscapes) {
59+
String escaped = noBackslashEscapes ? value : value.replace("\\", "\\\\");
60+
return "\"" + escaped.replace("\"", "\"\"") + "\"";
61+
}
62+
5763
public static List<String> splitMultiStmts(String sql) {
5864
DorisSqlSeparatorLexer lexer = new DorisSqlSeparatorLexer(CharStreams.fromString(sql));
5965
CommonTokenStream tokenStream = new CommonTokenStream(lexer);

0 commit comments

Comments
 (0)