1919
2020#include < fmt/format.h>
2121
22+ #include " agent/be_exec_version_manager.h"
2223#include " common/consts.h"
2324#include " common/exception.h"
2425#include " common/logging.h"
2526#include " core/block/block.h"
2627#include " core/column/column_nullable.h"
28+ #include " core/column/column_string.h"
29+ #include " core/column/column_struct.h"
2730#include " core/column/column_vector.h"
31+ #include " core/data_type/data_type_nullable.h"
32+ #include " core/data_type/data_type_struct.h"
2833#include " exec/sink/sink_common.h"
2934#include " exec/sink/viceberg_delete_sink.h"
3035#include " exec/sink/writer/iceberg/viceberg_table_writer.h"
@@ -52,8 +57,10 @@ Status VIcebergMergeSink::init_properties(ObjectPool* pool, const RowDescriptor&
5257
5358 _table_writer = std::make_unique<VIcebergTableWriter>(_table_sink, _table_output_expr_ctxs,
5459 nullptr , nullptr );
60+ _table_writer->defer_file_cleanup_until_outer_close ();
5561 _delete_writer = std::make_unique<VIcebergDeleteSink>(_delete_sink, _delete_output_expr_ctxs,
5662 nullptr , nullptr );
63+ _delete_writer->defer_file_cleanup_until_outer_close ();
5764 RETURN_IF_ERROR (_table_writer->init_properties (pool, row_desc));
5865 RETURN_IF_ERROR (_delete_writer->init_properties (pool));
5966 return Status::OK ();
@@ -65,6 +72,14 @@ Status VIcebergMergeSink::open(RuntimeState* state, RuntimeProfile* profile) {
6572 _written_rows_counter = ADD_COUNTER (profile, " RowsWritten" , TUnit::UNIT );
6673 _insert_rows_counter = ADD_COUNTER (profile, " InsertRows" , TUnit::UNIT );
6774 _delete_rows_counter = ADD_COUNTER (profile, " DeleteRows" , TUnit::UNIT );
75+ // The query-wide version keeps validation all-or-nothing during a rolling BE upgrade.
76+ _require_merge_cardinality_check =
77+ _require_merge_cardinality_check &&
78+ state->be_exec_version () >= SUPPORT_ICEBERG_MERGE_CARDINALITY_VERSION ;
79+ if (_require_merge_cardinality_check) {
80+ _matched_row_id_state_bytes_counter =
81+ ADD_COUNTER (profile, " MatchedRowIdStateBytes" , TUnit::BYTES );
82+ }
6883 _send_data_timer = ADD_TIMER (profile, " SendDataTime" );
6984 _open_timer = ADD_TIMER (profile, " OpenTime" );
7085 _close_timer = ADD_TIMER (profile, " CloseTime" );
@@ -94,8 +109,6 @@ Status VIcebergMergeSink::write(RuntimeState* state, Block& block) {
94109 return Status::OK ();
95110 }
96111
97- _row_count += output_block.rows ();
98-
99112 if (_operation_idx < 0 || _row_id_idx < 0 ) {
100113 return Status::InternalError (" Iceberg merge sink missing operation/row_id columns" );
101114 }
@@ -120,17 +133,26 @@ Status VIcebergMergeSink::write(RuntimeState* state, Block& block) {
120133 if (delete_op) {
121134 delete_filter[i] = 1 ;
122135 has_delete = true ;
123- ++_delete_row_count;
124136 ++delete_rows;
125137 }
126138 if (insert_op) {
127139 insert_filter[i] = 1 ;
128140 has_insert = true ;
129- ++_insert_row_count;
130141 ++insert_rows;
131142 }
132143 }
133144
145+ if (_require_merge_cardinality_check) {
146+ // The physical sink hashes matched rows by row_id, so exact state retained across blocks
147+ // enforces SQL MERGE cardinality for the whole query without changing UPDATE semantics.
148+ RETURN_IF_ERROR (_validate_matched_row_ids (output_block, delete_filter.data ()));
149+ COUNTER_SET (_matched_row_id_state_bytes_counter,
150+ static_cast <int64_t >(_matched_row_id_state_size));
151+ }
152+ _row_count += output_block.rows ();
153+ _delete_row_count += delete_rows;
154+ _insert_row_count += insert_rows;
155+
134156 bool skip_io = false ;
135157#ifdef BE_TEST
136158 skip_io = _skip_io;
@@ -167,6 +189,105 @@ Status VIcebergMergeSink::write(RuntimeState* state, Block& block) {
167189 return Status::OK ();
168190}
169191
192+ Status VIcebergMergeSink::_validate_matched_row_ids (const Block& block,
193+ const uint8_t * delete_filter) {
194+ const auto & row_id = block.get_by_position (_row_id_idx);
195+ const IColumn* row_id_data = row_id.column .get ();
196+ const IDataType* row_id_type = row_id.type .get ();
197+ const auto * nullable_row_id = check_and_get_column<ColumnNullable>(row_id_data);
198+ if (nullable_row_id != nullptr ) {
199+ row_id_data = nullable_row_id->get_nested_column_ptr ().get ();
200+ }
201+ if (const auto * nullable_type = check_and_get_data_type<DataTypeNullable>(row_id_type)) {
202+ row_id_type = nullable_type->get_nested_type ().get ();
203+ }
204+
205+ const auto * struct_column = check_and_get_column<ColumnStruct>(row_id_data);
206+ const auto * struct_type = check_and_get_data_type<DataTypeStruct>(row_id_type);
207+ if (struct_column == nullptr || struct_type == nullptr ) {
208+ return Status::InternalError (" Iceberg merge row_id column is not a struct" );
209+ }
210+
211+ int file_path_idx = -1 ;
212+ int row_position_idx = -1 ;
213+ const auto & field_names = struct_type->get_element_names ();
214+ for (size_t i = 0 ; i < field_names.size (); ++i) {
215+ std::string field_name = doris::to_lower (field_names[i]);
216+ if (field_name == " file_path" ) {
217+ file_path_idx = static_cast <int >(i);
218+ } else if (field_name == " row_position" ) {
219+ row_position_idx = static_cast <int >(i);
220+ }
221+ }
222+ if (file_path_idx < 0 || row_position_idx < 0 ) {
223+ return Status::InternalError (
224+ " Iceberg merge row_id must contain file_path and row_position fields" );
225+ }
226+
227+ const auto & file_path_column = struct_column->get_column_ptr (file_path_idx);
228+ const auto & row_position_column = struct_column->get_column_ptr (row_position_idx);
229+ const auto * nullable_file_path = check_and_get_column<ColumnNullable>(file_path_column.get ());
230+ const auto * nullable_row_position =
231+ check_and_get_column<ColumnNullable>(row_position_column.get ());
232+ const auto * file_paths =
233+ check_and_get_column<ColumnString>(remove_nullable (file_path_column).get ());
234+ const auto * row_positions = check_and_get_column<ColumnVector<TYPE_BIGINT >>(
235+ remove_nullable (row_position_column).get ());
236+ if (file_paths == nullptr || row_positions == nullptr ) {
237+ return Status::InternalError (" Iceberg merge row_id fields have incorrect types" );
238+ }
239+
240+ std::map<roaring::Roaring64Map*, size_t > touched_bitmap_sizes;
241+ for (size_t i = 0 ; i < block.rows (); ++i) {
242+ if (delete_filter[i] == 0 ) {
243+ continue ;
244+ }
245+ if ((nullable_row_id != nullptr && nullable_row_id->is_null_at (i)) ||
246+ (nullable_file_path != nullptr && nullable_file_path->is_null_at (i)) ||
247+ (nullable_row_position != nullptr && nullable_row_position->is_null_at (i))) {
248+ return Status::InternalError (" Iceberg merge matched row_id cannot be null" );
249+ }
250+
251+ int64_t row_position = row_positions->get_element (i);
252+ if (row_position < 0 ) {
253+ return Status::InternalError (" Invalid row_position {} in Iceberg merge row_id" ,
254+ row_position);
255+ }
256+ // Intern each file path once and keep exact positions in a compressed bitmap; retaining a
257+ // full path string per matched row makes MERGE memory grow with path_length * row_count.
258+ auto [file_it, inserted] =
259+ _matched_row_positions.try_emplace (file_paths->get_data_at (i).to_string ());
260+ auto * positions = &file_it->second ;
261+ auto touched_it = touched_bitmap_sizes.find (positions);
262+ if (touched_it == touched_bitmap_sizes.end ()) {
263+ touched_it = touched_bitmap_sizes.emplace (positions, positions->getSizeInBytes ()).first ;
264+ }
265+ if (inserted) {
266+ _matched_row_id_state_size +=
267+ sizeof (std::pair<const std::string, roaring::Roaring64Map>);
268+ _matched_row_id_state_size += file_it->first .capacity ();
269+ _matched_row_id_state_size += touched_it->second ;
270+ }
271+ if (!positions->addChecked (static_cast <uint64_t >(row_position))) {
272+ return Status::InvalidArgument (
273+ " Iceberg MERGE failed because multiple source rows matched the same target "
274+ " row" );
275+ }
276+ }
277+
278+ // Measure only bitmaps touched by this block; rescanning all retained files on every write
279+ // makes a many-file MERGE quadratic in the number of input blocks.
280+ for (const auto & [positions, previous_size] : touched_bitmap_sizes) {
281+ size_t current_size = positions->getSizeInBytes ();
282+ if (current_size >= previous_size) {
283+ _matched_row_id_state_size += current_size - previous_size;
284+ } else {
285+ _matched_row_id_state_size -= previous_size - current_size;
286+ }
287+ }
288+ return Status::OK ();
289+ }
290+
170291Status VIcebergMergeSink::close (Status close_status) {
171292 SCOPED_TIMER (_close_timer);
172293
@@ -201,10 +322,14 @@ Status VIcebergMergeSink::close(Status close_status) {
201322 COUNTER_SET (_delete_rows_counter, static_cast <int64_t >(_delete_row_count));
202323 }
203324
204- if (!table_status.ok ()) {
205- return table_status;
325+ Status result_status = table_status.ok () ? delete_status : table_status;
326+ if (_table_writer) {
327+ _table_writer->finish_deferred_file_cleanup (result_status);
328+ }
329+ if (_delete_writer) {
330+ _delete_writer->finish_deferred_file_cleanup (result_status);
206331 }
207- return delete_status ;
332+ return result_status ;
208333}
209334
210335Status VIcebergMergeSink::_build_inner_sinks () {
@@ -213,6 +338,9 @@ Status VIcebergMergeSink::_build_inner_sinks() {
213338 }
214339
215340 const auto & merge_sink = _t_sink.iceberg_merge_sink ;
341+ // Missing means an old FE plan, which predates SQL MERGE cardinality validation.
342+ _require_merge_cardinality_check = merge_sink.__isset .require_merge_cardinality_check &&
343+ merge_sink.require_merge_cardinality_check ;
216344
217345 TIcebergTableSink table_sink;
218346 if (merge_sink.__isset .db_name ) {
0 commit comments