Skip to content

Commit 8305fe7

Browse files
authored
[fix](iceberg) Enforce external write correctness (#66112)
## Proposed changes - Validate unsupported CTAS sinks before publishing table metadata, so setup failures cannot leave an unusable table or remove a concurrent replacement. - Preserve nullable string wrappers in truncate partition transforms. - Enforce Iceberg MERGE cardinality routing independently of `enable_strict_consistency_dml`. - Detect duplicate target matches with file-path-interned Roaring bitmaps and expose retained validation-state bytes in the sink profile. - Activate the negative regressions for the three retained Jira fixes and align the duplicate-match oracle with the emitted diagnostic. ## Test - Full build: `./build.sh --fe --be -j 48` - FE: `CreateTableCommandTest`, `IcebergDDLAndDMLPlanTest` (19 tests) - BE: `VIcebergMergeSinkTest` (8 tests, including 100,000 matched rows) - Regression: `test_paimon_ctas_atomicity_negative` (1 suite, passed) - Iceberg end-to-end SQL: duplicate-source MERGE rejected atomically; nullable/merge truncate writes and logical rows verified; physical partition values verified from the same Iceberg tables through Spark metadata. The Doris-side `$partitions`/`$snapshots` checks in the Iceberg suites currently hit an unrelated JNI scanner initialization abort on the current master test binary; the equivalent table state and metadata assertions above passed.
1 parent 204a1f0 commit 8305fe7

38 files changed

Lines changed: 1175 additions & 122 deletions

File tree

‎be/src/agent/be_exec_version_manager.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,8 +124,10 @@ void BeExecVersionManager::check_function_compatibility(int current_be_exec_vers
124124

125125
// 10: start from doris 4.0.3
126126
// a. use new fixed object serialization way.
127+
// 11: start from master
128+
// a. enforce Iceberg SQL MERGE cardinality only when every executing BE supports it.
127129

128-
const int BeExecVersionManager::max_be_exec_version = 10;
130+
const int BeExecVersionManager::max_be_exec_version = 11;
129131
const int BeExecVersionManager::min_be_exec_version = 0;
130132
std::map<std::string, std::set<int>> BeExecVersionManager::_function_change_map {};
131133
std::set<std::string> BeExecVersionManager::_function_restrict_map;

‎be/src/agent/be_exec_version_manager.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
namespace doris {
2727

2828
constexpr inline int USE_NEW_FIXED_OBJECT_SERIALIZATION_VERSION = 10;
29+
constexpr inline int SUPPORT_ICEBERG_MERGE_CARDINALITY_VERSION = 11;
2930

3031
class BeExecVersionManager {
3132
public:

‎be/src/exec/sink/viceberg_delete_sink.cpp‎

Lines changed: 46 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
#include "format/transformer/vfile_format_transformer.h"
4141
#include "io/file_factory.h"
4242
#include "runtime/runtime_state.h"
43+
#include "util/debug_points.h"
4344
#include "util/slice.h"
4445
#include "util/string_util.h"
4546
#include "util/uid_util.h"
@@ -252,17 +253,26 @@ Status VIcebergDeleteSink::close(Status close_status) {
252253
if (!close_status.ok()) {
253254
LOG(WARNING) << fmt::format("VIcebergDeleteSink close with error: {}",
254255
close_status.to_string());
256+
_cleanup_created_files();
255257
return close_status;
256258
}
257259

260+
Status write_status = Status::OK();
261+
DBUG_EXECUTE_IF("VIcebergDeleteSink.close.inject_failure", {
262+
write_status = Status::InternalError("injected Iceberg delete close failure");
263+
});
264+
258265
if (_delete_type == TFileContent::POSITION_DELETES && !_file_deletions.empty()) {
259266
SCOPED_TIMER(_write_delete_files_timer);
260-
if (_format_version >= 3) {
261-
RETURN_IF_ERROR(_write_deletion_vector_files(_file_deletions));
262-
} else {
263-
RETURN_IF_ERROR(_write_position_delete_files(_file_deletions));
267+
if (write_status.ok()) {
268+
write_status = _format_version >= 3 ? _write_deletion_vector_files(_file_deletions)
269+
: _write_position_delete_files(_file_deletions);
264270
}
265271
}
272+
if (!write_status.ok()) {
273+
_cleanup_created_files();
274+
return write_status;
275+
}
266276

267277
// Update counters
268278
if (_delete_file_count_counter) {
@@ -278,9 +288,33 @@ Status VIcebergDeleteSink::close(Status close_status) {
278288
}
279289
}
280290

291+
if (!_defer_file_cleanup_until_outer_close) {
292+
_created_files.clear();
293+
}
294+
281295
return Status::OK();
282296
}
283297

298+
void VIcebergDeleteSink::finish_deferred_file_cleanup(Status outer_status) {
299+
if (!outer_status.ok()) {
300+
_cleanup_created_files();
301+
} else {
302+
_created_files.clear();
303+
}
304+
_defer_file_cleanup_until_outer_close = false;
305+
}
306+
307+
void VIcebergDeleteSink::_cleanup_created_files() {
308+
for (const auto& [fs, path] : _created_files) {
309+
Status delete_status = fs->delete_file(path);
310+
if (!delete_status.ok() && !delete_status.is<ErrorCode::NOT_FOUND>()) {
311+
LOG(WARNING) << fmt::format("Failed to delete Iceberg delete file {}: {}", path,
312+
delete_status.to_string());
313+
}
314+
}
315+
_created_files.clear();
316+
}
317+
284318
int VIcebergDeleteSink::_get_row_id_column_index(const Block& block) {
285319
// Find __DORIS_ICEBERG_ROWID_COL__ column in block
286320
for (size_t i = 0; i < block.columns(); ++i) {
@@ -448,9 +482,13 @@ Status VIcebergDeleteSink::_write_position_delete_files(
448482
}
449483

450484
// Open writer
451-
RETURN_IF_ERROR(writer->open(_state, _state->runtime_profile(),
452-
_position_delete_output_expr_ctxs, column_names, _hadoop_conf,
453-
_file_type, _broker_addresses));
485+
Status open_status =
486+
writer->open(_state, _state->runtime_profile(), _position_delete_output_expr_ctxs,
487+
column_names, _hadoop_conf, _file_type, _broker_addresses);
488+
if (writer->file_system() != nullptr) {
489+
_created_files.emplace_back(writer->file_system(), delete_file_path);
490+
}
491+
RETURN_IF_ERROR(open_status);
454492

455493
// Build block with (file_path, pos) columns
456494
std::vector<int64_t> positions;
@@ -669,6 +707,7 @@ Status VIcebergDeleteSink::_write_puffin_file(const std::string& puffin_path,
669707
auto fs = DORIS_TRY(FileFactory::create_fs(fs_properties, file_description));
670708
io::FileWriterOptions file_writer_options = {.used_by_s3_committer = false};
671709
io::FileWriterPtr file_writer;
710+
_created_files.emplace_back(fs, puffin_path);
672711
RETURN_IF_ERROR(fs->create_file(file_description.path, &file_writer, &file_writer_options));
673712

674713
constexpr char PUFFIN_MAGIC[] = {'\x50', '\x46', '\x41', '\x31'};

‎be/src/exec/sink/viceberg_delete_sink.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,10 @@ class VIcebergDeleteSink final : public AsyncResultWriter {
7777

7878
Status close(Status) override;
7979

80+
void defer_file_cleanup_until_outer_close() { _defer_file_cleanup_until_outer_close = true; }
81+
82+
void finish_deferred_file_cleanup(Status outer_status);
83+
8084
private:
8185
/**
8286
* Extract $row_id column from block and group by file_path.
@@ -129,6 +133,7 @@ class VIcebergDeleteSink final : public AsyncResultWriter {
129133
const std::vector<int64_t>& positions, Block& output_block);
130134
Status _init_position_delete_output_exprs();
131135
std::string _get_file_extension() const;
136+
void _cleanup_created_files();
132137

133138
TDataSink _t_sink;
134139
RuntimeState* _state = nullptr;
@@ -141,6 +146,9 @@ class VIcebergDeleteSink final : public AsyncResultWriter {
141146

142147
// Collected commit data from all writers
143148
std::vector<TIcebergCommitData> _commit_data_list;
149+
// MERGE owns both data and delete objects until both inner sinks close successfully.
150+
std::vector<std::pair<std::shared_ptr<io::FileSystem>, std::string>> _created_files;
151+
bool _defer_file_cleanup_until_outer_close = false;
144152
// TODO: All deletions are held in memory until close(). Consider flushing
145153
// per-file when the upstream guarantees file_path ordering, or flushing
146154
// when estimated memory exceeds a threshold, to reduce peak memory usage.

‎be/src/exec/sink/viceberg_merge_sink.cpp‎

Lines changed: 135 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,17 @@
1919

2020
#include <fmt/format.h>
2121

22+
#include "agent/be_exec_version_manager.h"
2223
#include "common/consts.h"
2324
#include "common/exception.h"
2425
#include "common/logging.h"
2526
#include "core/block/block.h"
2627
#include "core/column/column_nullable.h"
28+
#include "core/column/column_string.h"
29+
#include "core/column/column_struct.h"
2730
#include "core/column/column_vector.h"
31+
#include "core/data_type/data_type_nullable.h"
32+
#include "core/data_type/data_type_struct.h"
2833
#include "exec/sink/sink_common.h"
2934
#include "exec/sink/viceberg_delete_sink.h"
3035
#include "exec/sink/writer/iceberg/viceberg_table_writer.h"
@@ -52,8 +57,10 @@ Status VIcebergMergeSink::init_properties(ObjectPool* pool, const RowDescriptor&
5257

5358
_table_writer = std::make_unique<VIcebergTableWriter>(_table_sink, _table_output_expr_ctxs,
5459
nullptr, nullptr);
60+
_table_writer->defer_file_cleanup_until_outer_close();
5561
_delete_writer = std::make_unique<VIcebergDeleteSink>(_delete_sink, _delete_output_expr_ctxs,
5662
nullptr, nullptr);
63+
_delete_writer->defer_file_cleanup_until_outer_close();
5764
RETURN_IF_ERROR(_table_writer->init_properties(pool, row_desc));
5865
RETURN_IF_ERROR(_delete_writer->init_properties(pool));
5966
return Status::OK();
@@ -65,6 +72,14 @@ Status VIcebergMergeSink::open(RuntimeState* state, RuntimeProfile* profile) {
6572
_written_rows_counter = ADD_COUNTER(profile, "RowsWritten", TUnit::UNIT);
6673
_insert_rows_counter = ADD_COUNTER(profile, "InsertRows", TUnit::UNIT);
6774
_delete_rows_counter = ADD_COUNTER(profile, "DeleteRows", TUnit::UNIT);
75+
// The query-wide version keeps validation all-or-nothing during a rolling BE upgrade.
76+
_require_merge_cardinality_check =
77+
_require_merge_cardinality_check &&
78+
state->be_exec_version() >= SUPPORT_ICEBERG_MERGE_CARDINALITY_VERSION;
79+
if (_require_merge_cardinality_check) {
80+
_matched_row_id_state_bytes_counter =
81+
ADD_COUNTER(profile, "MatchedRowIdStateBytes", TUnit::BYTES);
82+
}
6883
_send_data_timer = ADD_TIMER(profile, "SendDataTime");
6984
_open_timer = ADD_TIMER(profile, "OpenTime");
7085
_close_timer = ADD_TIMER(profile, "CloseTime");
@@ -94,8 +109,6 @@ Status VIcebergMergeSink::write(RuntimeState* state, Block& block) {
94109
return Status::OK();
95110
}
96111

97-
_row_count += output_block.rows();
98-
99112
if (_operation_idx < 0 || _row_id_idx < 0) {
100113
return Status::InternalError("Iceberg merge sink missing operation/row_id columns");
101114
}
@@ -120,17 +133,26 @@ Status VIcebergMergeSink::write(RuntimeState* state, Block& block) {
120133
if (delete_op) {
121134
delete_filter[i] = 1;
122135
has_delete = true;
123-
++_delete_row_count;
124136
++delete_rows;
125137
}
126138
if (insert_op) {
127139
insert_filter[i] = 1;
128140
has_insert = true;
129-
++_insert_row_count;
130141
++insert_rows;
131142
}
132143
}
133144

145+
if (_require_merge_cardinality_check) {
146+
// The physical sink hashes matched rows by row_id, so exact state retained across blocks
147+
// enforces SQL MERGE cardinality for the whole query without changing UPDATE semantics.
148+
RETURN_IF_ERROR(_validate_matched_row_ids(output_block, delete_filter.data()));
149+
COUNTER_SET(_matched_row_id_state_bytes_counter,
150+
static_cast<int64_t>(_matched_row_id_state_size));
151+
}
152+
_row_count += output_block.rows();
153+
_delete_row_count += delete_rows;
154+
_insert_row_count += insert_rows;
155+
134156
bool skip_io = false;
135157
#ifdef BE_TEST
136158
skip_io = _skip_io;
@@ -167,6 +189,105 @@ Status VIcebergMergeSink::write(RuntimeState* state, Block& block) {
167189
return Status::OK();
168190
}
169191

192+
Status VIcebergMergeSink::_validate_matched_row_ids(const Block& block,
193+
const uint8_t* delete_filter) {
194+
const auto& row_id = block.get_by_position(_row_id_idx);
195+
const IColumn* row_id_data = row_id.column.get();
196+
const IDataType* row_id_type = row_id.type.get();
197+
const auto* nullable_row_id = check_and_get_column<ColumnNullable>(row_id_data);
198+
if (nullable_row_id != nullptr) {
199+
row_id_data = nullable_row_id->get_nested_column_ptr().get();
200+
}
201+
if (const auto* nullable_type = check_and_get_data_type<DataTypeNullable>(row_id_type)) {
202+
row_id_type = nullable_type->get_nested_type().get();
203+
}
204+
205+
const auto* struct_column = check_and_get_column<ColumnStruct>(row_id_data);
206+
const auto* struct_type = check_and_get_data_type<DataTypeStruct>(row_id_type);
207+
if (struct_column == nullptr || struct_type == nullptr) {
208+
return Status::InternalError("Iceberg merge row_id column is not a struct");
209+
}
210+
211+
int file_path_idx = -1;
212+
int row_position_idx = -1;
213+
const auto& field_names = struct_type->get_element_names();
214+
for (size_t i = 0; i < field_names.size(); ++i) {
215+
std::string field_name = doris::to_lower(field_names[i]);
216+
if (field_name == "file_path") {
217+
file_path_idx = static_cast<int>(i);
218+
} else if (field_name == "row_position") {
219+
row_position_idx = static_cast<int>(i);
220+
}
221+
}
222+
if (file_path_idx < 0 || row_position_idx < 0) {
223+
return Status::InternalError(
224+
"Iceberg merge row_id must contain file_path and row_position fields");
225+
}
226+
227+
const auto& file_path_column = struct_column->get_column_ptr(file_path_idx);
228+
const auto& row_position_column = struct_column->get_column_ptr(row_position_idx);
229+
const auto* nullable_file_path = check_and_get_column<ColumnNullable>(file_path_column.get());
230+
const auto* nullable_row_position =
231+
check_and_get_column<ColumnNullable>(row_position_column.get());
232+
const auto* file_paths =
233+
check_and_get_column<ColumnString>(remove_nullable(file_path_column).get());
234+
const auto* row_positions = check_and_get_column<ColumnVector<TYPE_BIGINT>>(
235+
remove_nullable(row_position_column).get());
236+
if (file_paths == nullptr || row_positions == nullptr) {
237+
return Status::InternalError("Iceberg merge row_id fields have incorrect types");
238+
}
239+
240+
std::map<roaring::Roaring64Map*, size_t> touched_bitmap_sizes;
241+
for (size_t i = 0; i < block.rows(); ++i) {
242+
if (delete_filter[i] == 0) {
243+
continue;
244+
}
245+
if ((nullable_row_id != nullptr && nullable_row_id->is_null_at(i)) ||
246+
(nullable_file_path != nullptr && nullable_file_path->is_null_at(i)) ||
247+
(nullable_row_position != nullptr && nullable_row_position->is_null_at(i))) {
248+
return Status::InternalError("Iceberg merge matched row_id cannot be null");
249+
}
250+
251+
int64_t row_position = row_positions->get_element(i);
252+
if (row_position < 0) {
253+
return Status::InternalError("Invalid row_position {} in Iceberg merge row_id",
254+
row_position);
255+
}
256+
// Intern each file path once and keep exact positions in a compressed bitmap; retaining a
257+
// full path string per matched row makes MERGE memory grow with path_length * row_count.
258+
auto [file_it, inserted] =
259+
_matched_row_positions.try_emplace(file_paths->get_data_at(i).to_string());
260+
auto* positions = &file_it->second;
261+
auto touched_it = touched_bitmap_sizes.find(positions);
262+
if (touched_it == touched_bitmap_sizes.end()) {
263+
touched_it = touched_bitmap_sizes.emplace(positions, positions->getSizeInBytes()).first;
264+
}
265+
if (inserted) {
266+
_matched_row_id_state_size +=
267+
sizeof(std::pair<const std::string, roaring::Roaring64Map>);
268+
_matched_row_id_state_size += file_it->first.capacity();
269+
_matched_row_id_state_size += touched_it->second;
270+
}
271+
if (!positions->addChecked(static_cast<uint64_t>(row_position))) {
272+
return Status::InvalidArgument(
273+
"Iceberg MERGE failed because multiple source rows matched the same target "
274+
"row");
275+
}
276+
}
277+
278+
// Measure only bitmaps touched by this block; rescanning all retained files on every write
279+
// makes a many-file MERGE quadratic in the number of input blocks.
280+
for (const auto& [positions, previous_size] : touched_bitmap_sizes) {
281+
size_t current_size = positions->getSizeInBytes();
282+
if (current_size >= previous_size) {
283+
_matched_row_id_state_size += current_size - previous_size;
284+
} else {
285+
_matched_row_id_state_size -= previous_size - current_size;
286+
}
287+
}
288+
return Status::OK();
289+
}
290+
170291
Status VIcebergMergeSink::close(Status close_status) {
171292
SCOPED_TIMER(_close_timer);
172293

@@ -201,10 +322,14 @@ Status VIcebergMergeSink::close(Status close_status) {
201322
COUNTER_SET(_delete_rows_counter, static_cast<int64_t>(_delete_row_count));
202323
}
203324

204-
if (!table_status.ok()) {
205-
return table_status;
325+
Status result_status = table_status.ok() ? delete_status : table_status;
326+
if (_table_writer) {
327+
_table_writer->finish_deferred_file_cleanup(result_status);
328+
}
329+
if (_delete_writer) {
330+
_delete_writer->finish_deferred_file_cleanup(result_status);
206331
}
207-
return delete_status;
332+
return result_status;
208333
}
209334

210335
Status VIcebergMergeSink::_build_inner_sinks() {
@@ -213,6 +338,9 @@ Status VIcebergMergeSink::_build_inner_sinks() {
213338
}
214339

215340
const auto& merge_sink = _t_sink.iceberg_merge_sink;
341+
// Missing means an old FE plan, which predates SQL MERGE cardinality validation.
342+
_require_merge_cardinality_check = merge_sink.__isset.require_merge_cardinality_check &&
343+
merge_sink.require_merge_cardinality_check;
216344

217345
TIcebergTableSink table_sink;
218346
if (merge_sink.__isset.db_name) {

0 commit comments

Comments
 (0)