Skip to content

Commit 7604fbf

Browse files
authored
[fix](be) Fix Iceberg row-level deletes across schema evolution (apache#65502)
Problem Summary: Iceberg row-level delete handling had correctness and efficiency gaps across the V1 and V2 file scanners: FileScannerV2 applied delete expressions with ordinary predicate polarity, keeping rows whose delete predicate evaluated to true instead of removing them. Equality-delete keys could not be resolved when they were hidden from the query projection, renamed through Iceberg name mapping, absent from files written with older schemas, or stored in files without physical field ids. Missing equality keys were rejected instead of being materialized with their Iceberg initial default, or NULL when no initial default exists. Binary-like initial defaults could lose bytes in string transport, timestamp defaults could be parsed with the wrong mapping, and time-travel scans could use metadata from the latest schema instead of the selected snapshot schema. Equality and position delete files shared by multiple data splits were reparsed for every split, and position bounds were not applied. Complex-column rematerialization could trust an incomplete nested descriptor and produce a child type whose nullability disagreed with its parent Struct type. Applying that correction to Array/Map wrapper children could also trigger a fatal child-name mismatch. This change: Builds an explicit keep filter by inverting delete matches and combines multiple delete files with the correct union semantics. Sends complete Iceberg schema metadata from FE to BE, including historical name mapping and losslessly encoded initial defaults from the selected snapshot schema. Resolves equality keys consistently by field id or BY_NAME mapping in both V1 and V2 scanners, and materializes missing keys with their typed initial default or NULL. Supports all-literal equality predicates by materializing constant keys to the current batch size. Propagates TIMESTAMPTZ mapping to delete-file readers. Caches parsed equality and position delete data in SplitReadOptions.cache, with filesystem, path, and equality-field layout in the cache identity, and applies inclusive position bounds. Uses the parent Struct type as the authoritative contract for child nullability while preserving Array and Map wrapper semantics.
1 parent a37667d commit 7604fbf

28 files changed

Lines changed: 2712 additions & 269 deletions

‎be/src/format/table/iceberg_reader.cpp‎

Lines changed: 155 additions & 48 deletions
Large diffs are not rendered by default.

‎be/src/format/table/iceberg_reader.h‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -105,8 +105,9 @@ class IcebergParquetReader final : public IcebergReaderMixin<ParquetReader> {
105105
this->get_state(), this->_meta_cache);
106106
}
107107

108-
static ColumnIdResult _create_column_ids(const FieldDescriptor* field_desc,
109-
const TupleDescriptor* tuple_descriptor);
108+
static ColumnIdResult _create_column_ids(
109+
const FieldDescriptor* field_desc, const TupleDescriptor* tuple_descriptor,
110+
const std::shared_ptr<TableSchemaChangeHelper::Node>& table_info_node = nullptr);
110111

111112
private:
112113
Status _read_position_delete_file(const TFileRangeDesc* delete_range,
@@ -151,8 +152,9 @@ class IcebergOrcReader final : public IcebergReaderMixin<OrcReader> {
151152
this->get_io_ctx(), this->_meta_cache);
152153
}
153154

154-
static ColumnIdResult _create_column_ids(const orc::Type* orc_type,
155-
const TupleDescriptor* tuple_descriptor);
155+
static ColumnIdResult _create_column_ids(
156+
const orc::Type* orc_type, const TupleDescriptor* tuple_descriptor,
157+
const std::shared_ptr<TableSchemaChangeHelper::Node>& table_info_node = nullptr);
156158

157159
static const std::string ICEBERG_ORC_ATTRIBUTE;
158160

‎be/src/format/table/iceberg_reader_mixin.h‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,12 @@
1717

1818
#pragma once
1919

20+
#include <gen_cpp/ExternalTableSchema_types.h>
21+
2022
#include <cstddef>
2123
#include <cstdint>
2224
#include <memory>
25+
#include <ranges>
2326
#include <string>
2427
#include <unordered_map>
2528
#include <vector>
@@ -33,13 +36,15 @@
3336
#include "core/column/column_struct.h"
3437
#include "core/data_type/data_type_number.h"
3538
#include "core/data_type/data_type_string.h"
39+
#include "core/data_type/primitive_type.h"
3640
#include "format/generic_reader.h"
3741
#include "format/table/equality_delete.h"
3842
#include "format/table/iceberg_delete_file_reader_helper.h"
3943
#include "format/table/table_schema_change_helper.h"
4044
#include "runtime/runtime_profile.h"
4145
#include "runtime/runtime_state.h"
4246
#include "storage/olap_common.h"
47+
#include "util/url_coding.h"
4348

4449
namespace doris {
4550
class TIcebergDeleteFileDesc;
@@ -135,6 +140,20 @@ class IcebergReaderMixin : public BaseReader, public TableSchemaChangeHelper {
135140
return _position_delete_base(data_file_path, delete_files);
136141
}
137142

143+
void TEST_set_column_name_to_block_index(
144+
std::unordered_map<std::string, uint32_t>* column_name_to_block_index) {
145+
this->col_name_to_block_idx_ref() = column_name_to_block_index;
146+
}
147+
148+
Status TEST_register_missing_equality_delete_column(int32_t field_id, const std::string& name,
149+
const DataTypePtr& delete_key_type) {
150+
return _register_missing_equality_delete_column(field_id, name, delete_key_type);
151+
}
152+
153+
Status TEST_materialize_missing_equality_delete_columns(Block* block, size_t rows) {
154+
return _materialize_missing_equality_delete_columns(block, rows);
155+
}
156+
138157
protected:
139158
// ---- Hook implementations ----
140159

@@ -251,6 +270,11 @@ class IcebergReaderMixin : public BaseReader, public TableSchemaChangeHelper {
251270

252271
Status _expand_block_if_need(Block* block);
253272
Status _shrink_block_if_need(Block* block);
273+
Status _register_missing_equality_delete_column(int32_t field_id, const std::string& name,
274+
const DataTypePtr& delete_key_type);
275+
Status _materialize_missing_equality_delete_column(Block* block, const std::string& name,
276+
const ColumnPtr& value, size_t rows);
277+
Status _materialize_missing_equality_delete_columns(Block* block, size_t rows);
254278

255279
// Type aliases — must be defined before member function declarations that use them.
256280
using DeleteRows = std::vector<int64_t>;
@@ -352,6 +376,7 @@ class IcebergReaderMixin : public BaseReader, public TableSchemaChangeHelper {
352376
const DeletionVector* _iceberg_deletion_vector = nullptr;
353377
std::vector<std::string> _expand_col_names;
354378
std::vector<ColumnWithTypeAndName> _expand_columns;
379+
std::unordered_map<std::string, ColumnPtr> _missing_equality_delete_values;
355380
std::vector<std::string> _all_required_col_names;
356381
Fileformat _file_format = Fileformat::NONE;
357382

@@ -624,6 +649,11 @@ Status IcebergReaderMixin<BaseReader>::_expand_block_if_need(Block* block) {
624649
auto block_names = block->get_names();
625650
names.insert(block_names.begin(), block_names.end());
626651
for (auto& col : _expand_columns) {
652+
if (_missing_equality_delete_values.contains(col.name)) {
653+
// Missing equality keys are logical columns, not physical file columns. Add them only
654+
// after the base reader has established the batch row count.
655+
continue;
656+
}
627657
if (names.contains(col.name)) {
628658
return Status::InternalError("Wrong expand column '{}'", col.name);
629659
}
@@ -634,6 +664,118 @@ Status IcebergReaderMixin<BaseReader>::_expand_block_if_need(Block* block) {
634664
return Status::OK();
635665
}
636666

667+
template <typename BaseReader>
668+
Status IcebergReaderMixin<BaseReader>::_register_missing_equality_delete_column(
669+
int32_t field_id, const std::string& name, const DataTypePtr& delete_key_type) {
670+
DORIS_CHECK(delete_key_type != nullptr);
671+
const schema::external::TField* table_field = nullptr;
672+
const auto& scan_params = this->get_scan_params();
673+
const schema::external::TSchema* current_schema = nullptr;
674+
if (scan_params.__isset.history_schema_info && !scan_params.history_schema_info.empty()) {
675+
current_schema = &scan_params.history_schema_info.front();
676+
}
677+
if (current_schema != nullptr && scan_params.__isset.current_schema_id) {
678+
const auto schema_it = std::ranges::find_if(
679+
scan_params.history_schema_info, [&](const schema::external::TSchema& schema) {
680+
return schema.__isset.schema_id &&
681+
schema.schema_id == scan_params.current_schema_id;
682+
});
683+
if (schema_it != scan_params.history_schema_info.end()) {
684+
current_schema = &*schema_it;
685+
} else {
686+
current_schema = nullptr;
687+
}
688+
}
689+
if (current_schema != nullptr && current_schema->__isset.root_field) {
690+
for (const auto& field_ptr : current_schema->root_field.fields) {
691+
if (field_ptr.__isset.field_ptr && field_ptr.field_ptr != nullptr &&
692+
field_ptr.field_ptr->__isset.id && field_ptr.field_ptr->id == field_id) {
693+
table_field = field_ptr.field_ptr.get();
694+
break;
695+
}
696+
}
697+
}
698+
if (table_field == nullptr) {
699+
// A projected descriptor from an older FE may omit a hidden equality key. Without the
700+
// current Iceberg field metadata BE cannot distinguish a true NULL initial default from a
701+
// non-NULL default, so continuing would risk silently keeping or deleting wrong rows.
702+
return Status::InternalError(
703+
"Missing Iceberg schema metadata for equality-delete field id {}", field_id);
704+
}
705+
706+
Field value;
707+
if (table_field->__isset.initial_default_value) {
708+
const auto nested_type = remove_nullable(delete_key_type);
709+
const bool default_is_base64 = (table_field->__isset.initial_default_value_is_base64 &&
710+
table_field->initial_default_value_is_base64) ||
711+
(table_field->__isset.type &&
712+
thrift_to_type(table_field->type.type) == TYPE_VARBINARY);
713+
if (default_is_base64) {
714+
std::string decoded_default;
715+
if (!base64_decode(table_field->initial_default_value, &decoded_default)) {
716+
return Status::InvalidArgument(
717+
"Invalid Base64 Iceberg initial default for field {}", table_field->name);
718+
}
719+
if (nested_type->get_primitive_type() == TYPE_VARBINARY) {
720+
value = Field::create_field<TYPE_VARBINARY>(StringView(decoded_default));
721+
} else {
722+
DORIS_CHECK(is_string_type(nested_type->get_primitive_type()));
723+
value = Field::create_field<TYPE_STRING>(decoded_default);
724+
}
725+
} else {
726+
RETURN_IF_ERROR(nested_type->get_serde()->from_fe_string(
727+
table_field->initial_default_value, value));
728+
}
729+
}
730+
const bool inserted = _missing_equality_delete_values
731+
.emplace(name, delete_key_type->create_column_const(1, value))
732+
.second;
733+
DORIS_CHECK(inserted);
734+
this->register_synthesized_column_handler(
735+
name, [this, name](Block* block, size_t rows) -> Status {
736+
DORIS_CHECK(_missing_equality_delete_values.contains(name));
737+
return _materialize_missing_equality_delete_column(
738+
block, name, _missing_equality_delete_values.at(name), rows);
739+
});
740+
return Status::OK();
741+
}
742+
743+
template <typename BaseReader>
744+
Status IcebergReaderMixin<BaseReader>::_materialize_missing_equality_delete_column(
745+
Block* block, const std::string& name, const ColumnPtr& value, size_t rows) {
746+
if (!this->col_name_to_block_idx_ref()->contains(name)) {
747+
// ORC must not register a key that is absent from the file as a physical child. In that
748+
// case the reader block has no slot for the synthesized key, so append one here before
749+
// equality-delete filtering. MultiEqualityDelete requires a full, batch-sized column.
750+
const auto expand_col = std::ranges::find_if(
751+
_expand_columns,
752+
[&](const ColumnWithTypeAndName& col) { return col.name == name; });
753+
DORIS_CHECK(expand_col != _expand_columns.end());
754+
(*this->col_name_to_block_idx_ref())[name] = block->columns();
755+
block->insert({value->clone_resized(rows)->convert_to_full_column_if_const(),
756+
expand_col->type, name});
757+
return Status::OK();
758+
}
759+
const auto position = this->col_name_to_block_idx_ref()->at(name);
760+
DORIS_CHECK(position < block->columns());
761+
DORIS_CHECK(block->get_by_position(position).column->empty());
762+
// MultiEqualityDelete hashes each key column directly. Materialize the repeated default so
763+
// every key has the batch row count; a ColumnConst keeps only one nested value and therefore
764+
// cannot participate in the row-wise multi-column hash contract.
765+
block->get_by_position(position).column =
766+
value->clone_resized(rows)->convert_to_full_column_if_const();
767+
return Status::OK();
768+
}
769+
770+
template <typename BaseReader>
771+
Status IcebergReaderMixin<BaseReader>::_materialize_missing_equality_delete_columns(Block* block,
772+
size_t rows) {
773+
for (const auto& [name, value] : _missing_equality_delete_values) {
774+
RETURN_IF_ERROR(_materialize_missing_equality_delete_column(block, name, value, rows));
775+
}
776+
return Status::OK();
777+
}
778+
637779
template <typename BaseReader>
638780
Status IcebergReaderMixin<BaseReader>::_shrink_block_if_need(Block* block) {
639781
std::set<size_t> positions_to_erase;

‎be/src/format_v2/column_data.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,12 @@ struct ColumnDefinition {
258258
// Expression used to materialize missing/default/generated values when the column is not read
259259
// directly from the file.
260260
VExprContextSPtr default_expr = nullptr;
261+
// Table-format initial default normalized for transport from FE. Binary-like values use Base64
262+
// and set initial_default_value_is_base64 because they can map to STRING/CHAR or VARBINARY.
263+
// Unlike default_expr, this metadata is also available for hidden delete-predicate columns
264+
// that are absent from the query projection.
265+
std::optional<std::string> initial_default_value = std::nullopt;
266+
bool initial_default_value_is_base64 = false;
261267
// Partition columns are constants from split metadata and should not be matched against file
262268
// schema unless table-format logic explicitly asks for it.
263269
bool is_partition_key = false;

‎be/src/format_v2/column_mapper.cpp‎

Lines changed: 61 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,11 @@ std::string join_debug_strings(const std::vector<T>& values, Formatter formatter
230230

231231
} // namespace
232232

233+
const ColumnDefinition* find_column_by_name(const ColumnDefinition& table_column,
234+
const std::vector<ColumnDefinition>& file_schema) {
235+
return matcher_for_mode(TableColumnMappingMode::BY_NAME).find(table_column, file_schema);
236+
}
237+
233238
const Field* find_partition_value(const ColumnDefinition& table_column,
234239
const std::map<std::string, Field>& partition_values) {
235240
const auto find_by_name = [&](const std::string& name) -> const Field* {
@@ -1237,6 +1242,26 @@ static std::vector<ColumnDefinition> synthesize_complex_children_from_type(
12371242
return children;
12381243
}
12391244

1245+
static void align_struct_child_types_with_parent(const DataTypePtr& parent_type,
1246+
std::vector<ColumnDefinition>& children) {
1247+
const auto nested_parent_type = remove_nullable(parent_type);
1248+
DORIS_CHECK(nested_parent_type->get_primitive_type() == TYPE_STRUCT);
1249+
const auto type_children = synthesize_complex_children_from_type(parent_type);
1250+
for (auto& child : children) {
1251+
const auto type_child = std::ranges::find_if(
1252+
type_children, [&](const auto& candidate) { return candidate.name == child.name; });
1253+
DORIS_CHECK(type_child != type_children.end())
1254+
<< "Complex child '" << child.name
1255+
<< "' is absent from its parent table type: " << parent_type->get_name();
1256+
// The parent DataType is the authoritative output contract. Nested schema descriptors can
1257+
// omit child nullability even though the parent struct still declares Nullable(String).
1258+
// For example, the Iceberg full-schema-change case maps nullable `location` to `city`, but
1259+
// its child descriptor carries String. Keeping String here makes rematerialization strip
1260+
// the child's null map and creates Struct(String) under a Struct(Nullable(String)) type.
1261+
child.type = type_child->type;
1262+
}
1263+
}
1264+
12401265
static bool has_table_child_named(const std::vector<ColumnDefinition>& children,
12411266
std::string_view name) {
12421267
return std::ranges::any_of(children, [&](const ColumnDefinition& child) {
@@ -1245,8 +1270,7 @@ static bool has_table_child_named(const std::vector<ColumnDefinition>& children,
12451270
}
12461271

12471272
static void complete_required_complex_children_from_type(const DataTypePtr& type,
1248-
std::vector<ColumnDefinition>* children) {
1249-
DORIS_CHECK(children != nullptr);
1273+
std::vector<ColumnDefinition>& children) {
12501274
if (type == nullptr) {
12511275
return;
12521276
}
@@ -1261,8 +1285,8 @@ static void complete_required_complex_children_from_type(const DataTypePtr& type
12611285
// In that shape the scanner keeps the value stream readable, but the table projection can
12621286
// carry only the key child. Add the missing value child so recursive mapping can evolve the
12631287
// value type instead of letting TableReader cast old/new value structs directly.
1264-
if (has_table_child_named(*children, "key") && !has_table_child_named(*children, "value")) {
1265-
children->push_back(synthetic_child_definition("value", map_type->get_value_type(), 1));
1288+
if (has_table_child_named(children, "key") && !has_table_child_named(children, "value")) {
1289+
children.push_back(synthetic_child_definition("value", map_type->get_value_type(), 1));
12661290
}
12671291
break;
12681292
}
@@ -1279,6 +1303,37 @@ static void complete_required_complex_children_from_type(const DataTypePtr& type
12791303
}
12801304
}
12811305

1306+
struct PreparedTableChildren {
1307+
std::vector<ColumnDefinition> children;
1308+
bool synthesized_from_type = false;
1309+
};
1310+
1311+
static PreparedTableChildren prepare_table_children_for_mapping(
1312+
const ColumnDefinition& table_column, const DataTypePtr& file_type) {
1313+
PreparedTableChildren prepared {.children = table_column.children};
1314+
const auto nested_table_type = remove_nullable(table_column.type);
1315+
1316+
// Some scan paths, especially SELECT *, only carry the complete complex DataType for a table
1317+
// column and leave ColumnDefinition::children empty. Synthesize the hierarchy so recursive
1318+
// mapping can evolve nested fields instead of falling back to an invalid whole-column cast.
1319+
prepared.synthesized_from_type = prepared.children.empty() &&
1320+
is_complex_type(nested_table_type->get_primitive_type()) &&
1321+
!table_column.type->equals(*file_type);
1322+
if (prepared.synthesized_from_type) {
1323+
prepared.children = synthesize_complex_children_from_type(table_column.type);
1324+
} else if (!prepared.children.empty() && !table_column.type->equals(*file_type)) {
1325+
complete_required_complex_children_from_type(table_column.type, prepared.children);
1326+
}
1327+
1328+
if (!prepared.children.empty() && nested_table_type->get_primitive_type() == TYPE_STRUCT) {
1329+
// Struct children are table fields, so the parent Struct type is authoritative for their
1330+
// nullability. ARRAY and MAP children are format-level structural wrappers and keep the
1331+
// descriptor types used by their recursive mappings.
1332+
align_struct_child_types_with_parent(table_column.type, prepared.children);
1333+
}
1334+
return prepared;
1335+
}
1336+
12821337
static Status validate_file_schema_children(const ColumnDefinition& file_field) {
12831338
if (file_field.type == nullptr) {
12841339
return Status::InternalError("File column '{}' has null type", file_field.name);
@@ -2016,29 +2071,8 @@ Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_c
20162071
mapping->filter_conversion = direct_filter_conversion(*mapping);
20172072
mapping->child_mappings.clear();
20182073

2019-
auto table_children = table_column.children;
2020-
const auto nested_table_type = remove_nullable(mapping->table_type);
2021-
// Some scan paths, especially SELECT *, only carry the complete complex DataType for a table
2022-
// column and leave ColumnDefinition::children empty. If the file type is an older complex
2023-
// schema, treating this as a leaf mapping would make TableReader fall back to a plain CAST.
2024-
// That is invalid for evolved structs with different field counts.
2025-
//
2026-
// Example:
2027-
// table column type: Map(String, Struct(age, full_name, gender))
2028-
// old file type: Map(String, Struct(age, name))
2029-
// table children: empty
2030-
//
2031-
// Synthesize key/value/struct-field children from the table type so the normal recursive
2032-
// mapping path can rematerialize `name -> full_name` and fill missing `gender` with defaults,
2033-
// instead of trying to CAST Struct(age, name) to Struct(age, full_name, gender).
2034-
const bool synthesized_table_children =
2035-
table_children.empty() && is_complex_type(nested_table_type->get_primitive_type()) &&
2036-
!mapping->table_type->equals(*mapping->file_type);
2037-
if (synthesized_table_children) {
2038-
table_children = synthesize_complex_children_from_type(mapping->table_type);
2039-
} else if (!table_children.empty() && !mapping->table_type->equals(*mapping->file_type)) {
2040-
complete_required_complex_children_from_type(mapping->table_type, &table_children);
2041-
}
2074+
auto [table_children, synthesized_table_children] =
2075+
prepare_table_children_for_mapping(table_column, mapping->file_type);
20422076

20432077
if (!table_children.empty()) {
20442078
if (!is_complex_type(remove_nullable(mapping->file_type)->get_primitive_type())) {

0 commit comments

Comments
 (0)