Skip to content

Commit a597245

Browse files
committed
[fix](be) Release deletion vector rows on read errors
### What problem does this PR solve? Issue Number: None Related PR: #65351 Problem Summary: Deletion vector and position-delete cache builders allocated DeleteRows with raw new and returned nullptr on read or parse failures. Those error paths leaked the partially built DeleteRows object and left the failure behavior under-tested in the V1 Iceberg and Paimon readers. Use RAII while building DeleteRows and release ownership only after successful construction. Add UT coverage for V1 Iceberg reader deletion vector read failures, V1 Paimon Parquet/ORC deletion vector read failures, the shared Iceberg helper fault paths, and the format v2 TableReader deletion vector fault paths. ### Release note None ### Check List (For Author) - Test: Regression test / Unit Test - Added BE unit tests for V1/shared Iceberg deletion vector helper fault and stress paths. - Added BE unit tests for V1 Iceberg reader deletion vector read failure cache-builder path. - Added BE unit tests for V1 Paimon Parquet/ORC reader deletion vector read failure cache-builder paths. - Added BE unit tests for format v2 Iceberg deletion vector fault paths. - Extended Iceberg deletion vector regression to cover both scanner v1 and scanner v2 debug point fault injection. - Ran build-support/clang-format.sh locally. - Ran PATH=/usr/local/opt/llvm@16/bin:$PATH build-support/check-format.sh locally. - Ran git diff --check locally. - On [email protected] in /mnt/disk3/gabriel/Workspace/dev1/doris_pr65351_ut, ran ./run-be-ut.sh --run --filter="IcebergReaderTest.v1_deletion_vector_read_error_releases_cache_entry:PaimonDeletionVectorTest.V1*:IcebergDeleteFileReaderHelperTest.*:IcebergV2ReaderTest.IcebergDeletionVector*:IcebergV2ReaderTest.IcebergTableReader*DeletionVector*"; 24 tests passed. - Attempted build-support/run-clang-tidy.sh --build-dir be/cmake-build-debug-dev-perf locally with CLANG_TIDY_BINARY=/usr/local/opt/llvm@16/bin/clang-tidy; blocked by the local macOS bash 3.2 missing mapfile. - Behavior changed: No - Does this need documentation: No
1 parent b756496 commit a597245

10 files changed

Lines changed: 627 additions & 15 deletions

File tree

‎be/src/format/table/iceberg_delete_file_reader_helper.cpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
#include "io/hdfs_builder.h"
4545
#include "runtime/runtime_state.h"
4646
#include "storage/predicate/column_predicate.h"
47+
#include "util/debug_points.h"
4748

4849
namespace doris {
4950

@@ -301,6 +302,10 @@ Status read_iceberg_deletion_vector(const TIcebergDeleteFileDesc& delete_file,
301302
if (!delete_file.__isset.content_offset || !delete_file.__isset.content_size_in_bytes) {
302303
return Status::InternalError("Deletion vector is missing content offset or length");
303304
}
305+
DBUG_EXECUTE_IF("IcebergDeleteFileReader.read_deletion_vector.io_error",
306+
{ return Status::IOError("injected Iceberg deletion vector read failure"); });
307+
DBUG_EXECUTE_IF("IcebergDeleteFileReader.read_deletion_vector.should_stop",
308+
{ return Status::EndOfFile("stop read."); });
304309

305310
TFileRangeDesc delete_range = build_iceberg_delete_file_range(delete_file.path);
306311
if (options.fs_name != nullptr && !options.fs_name->empty()) {
@@ -316,7 +321,7 @@ Status read_iceberg_deletion_vector(const TIcebergDeleteFileDesc& delete_file,
316321
std::vector<char> buf(delete_range.size);
317322
RETURN_IF_ERROR(dv_reader.read_at(delete_range.start_offset,
318323
{buf.data(), cast_set<size_t>(delete_range.size)}));
319-
return decode_iceberg_deletion_vector_buffer(buf.data(), delete_range.size, rows_to_delete);
324+
return decode_deletion_vector_buffer(buf.data(), delete_range.size, rows_to_delete);
320325
}
321326

322327
Status decode_iceberg_deletion_vector_buffer(const char* buf, size_t buffer_size,

‎be/src/format/table/iceberg_reader_mixin.h‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
#include <cstddef>
2121
#include <cstdint>
22+
#include <memory>
2223
#include <string>
2324
#include <unordered_map>
2425
#include <vector>
@@ -108,6 +109,11 @@ class IcebergReaderMixin : public BaseReader, public TableSchemaChangeHelper {
108109
_create_topn_row_id_column_iterator = create_func;
109110
}
110111

112+
Status TEST_read_deletion_vector(const std::string& data_file_path,
113+
const TIcebergDeleteFileDesc& delete_file_desc) {
114+
return read_deletion_vector(data_file_path, delete_file_desc);
115+
}
116+
111117
protected:
112118
// ---- Hook implementations ----
113119

@@ -660,10 +666,10 @@ Status IcebergReaderMixin<BaseReader>::_position_delete_base(
660666
SCOPED_TIMER(_iceberg_profile.delete_rows_sort_time);
661667
_iceberg_delete_rows =
662668
_kv_cache->template get<DeleteRows>(data_file_path, [&]() -> DeleteRows* {
663-
auto* data_file_position_delete = new DeleteRows;
669+
auto data_file_position_delete = std::make_unique<DeleteRows>();
664670
_sort_delete_rows(delete_rows_array, num_delete_rows,
665671
*data_file_position_delete);
666-
return data_file_position_delete;
672+
return data_file_position_delete.release();
667673
});
668674
set_delete_rows();
669675
COUNTER_UPDATE(_iceberg_profile.num_delete_rows, num_delete_rows);
@@ -808,7 +814,7 @@ Status IcebergReaderMixin<BaseReader>::read_deletion_vector(
808814
_iceberg_delete_rows = _kv_cache->template get<DeleteRows>(
809815
build_iceberg_deletion_vector_cache_key(data_file_path, delete_file_desc),
810816
[&]() -> DeleteRows* {
811-
auto* delete_rows = new DeleteRows;
817+
auto delete_rows = std::make_unique<DeleteRows>();
812818

813819
roaring::Roaring64Map bitmap;
814820
IcebergDeleteFileReaderOptions options;
@@ -828,7 +834,7 @@ Status IcebergReaderMixin<BaseReader>::read_deletion_vector(
828834
delete_rows->push_back(*it);
829835
}
830836
COUNTER_UPDATE(_iceberg_profile.num_delete_rows, delete_rows->size());
831-
return delete_rows;
837+
return delete_rows.release();
832838
});
833839

834840
RETURN_IF_ERROR(create_status);

‎be/src/format/table/paimon_reader.cpp‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include <fmt/format.h>
2121

2222
#include <cstring>
23+
#include <memory>
2324
#include <vector>
2425

2526
#include "common/status.h"
@@ -134,7 +135,7 @@ Status PaimonOrcReader::_init_deletion_vector() {
134135
using DeleteRows = std::vector<int64_t>;
135136
_delete_rows = _kv_cache->get<DeleteRows>(
136137
build_paimon_deletion_vector_cache_key(deletion_file), [&]() -> DeleteRows* {
137-
auto* delete_rows = new DeleteRows;
138+
auto delete_rows = std::make_unique<DeleteRows>();
138139

139140
TFileRangeDesc delete_range;
140141
delete_range.__set_fs_name(get_scan_range().fs_name);
@@ -160,12 +161,12 @@ Status PaimonOrcReader::_init_deletion_vector() {
160161

161162
SCOPED_TIMER(_paimon_profile.parse_deletion_vector_time);
162163
create_status = decode_paimon_deletion_vector_buffer(buffer.data(), bytes_read,
163-
delete_rows);
164+
delete_rows.get());
164165
if (!create_status.ok()) [[unlikely]] {
165166
return nullptr;
166167
}
167168
COUNTER_UPDATE(_paimon_profile.num_delete_rows, delete_rows->size());
168-
return delete_rows;
169+
return delete_rows.release();
169170
});
170171
RETURN_IF_ERROR(create_status);
171172
if (!_delete_rows->empty()) [[likely]] {
@@ -233,7 +234,7 @@ Status PaimonParquetReader::_init_deletion_vector() {
233234
using DeleteRows = std::vector<int64_t>;
234235
_delete_rows = _kv_cache->get<DeleteRows>(
235236
build_paimon_deletion_vector_cache_key(deletion_file), [&]() -> DeleteRows* {
236-
auto* delete_rows = new DeleteRows;
237+
auto delete_rows = std::make_unique<DeleteRows>();
237238

238239
TFileRangeDesc delete_range;
239240
delete_range.__set_fs_name(get_scan_range().fs_name);
@@ -259,12 +260,12 @@ Status PaimonParquetReader::_init_deletion_vector() {
259260

260261
SCOPED_TIMER(_paimon_profile.parse_deletion_vector_time);
261262
create_status = decode_paimon_deletion_vector_buffer(buffer.data(), bytes_read,
262-
delete_rows);
263+
delete_rows.get());
263264
if (!create_status.ok()) [[unlikely]] {
264265
return nullptr;
265266
}
266267
COUNTER_UPDATE(_paimon_profile.num_delete_rows, delete_rows->size());
267-
return delete_rows;
268+
return delete_rows.release();
268269
});
269270
RETURN_IF_ERROR(create_status);
270271
if (!_delete_rows->empty()) [[likely]] {

‎be/src/format/table/paimon_reader.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ class PaimonOrcReader final : public OrcReader, public TableSchemaChangeHelper {
6464
}
6565
~PaimonOrcReader() final = default;
6666

67+
Status TEST_init_deletion_vector() { return _init_deletion_vector(); }
68+
6769
protected:
6870
Status on_before_init_reader(ReaderInitContext* ctx) override;
6971

@@ -109,6 +111,8 @@ class PaimonParquetReader final : public ParquetReader, public TableSchemaChange
109111
}
110112
~PaimonParquetReader() final = default;
111113

114+
Status TEST_init_deletion_vector() { return _init_deletion_vector(); }
115+
112116
protected:
113117
Status on_before_init_reader(ReaderInitContext* ctx) override;
114118

‎be/src/format_v2/table_reader.cpp‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include <gen_cpp/Types_types.h>
2323

2424
#include <algorithm>
25+
#include <memory>
2526
#include <ranges>
2627
#include <set>
2728
#include <sstream>
@@ -46,6 +47,7 @@
4647
#include "format_v2/native/native_reader.h"
4748
#include "format_v2/parquet/parquet_reader.h"
4849
#include "storage/segment/condition_cache.h"
50+
#include "util/debug_points.h"
4951
#include "util/string_util.h"
5052

5153
namespace doris::format {
@@ -765,7 +767,7 @@ Status TableReader::_parse_delete_predicates(const SplitReadOptions& options) {
765767
Status create_status = Status::OK();
766768

767769
_delete_rows = options.cache->get<DeleteRows>(desc.key, [&]() -> DeleteRows* {
768-
auto* delete_rows = new DeleteRows;
770+
auto delete_rows = std::make_unique<DeleteRows>();
769771

770772
DeletionVectorReader dv_reader(_runtime_state, _scanner_profile, *_scan_params, desc,
771773
_io_ctx.get());
@@ -776,19 +778,27 @@ Status TableReader::_parse_delete_predicates(const SplitReadOptions& options) {
776778

777779
size_t bytes_read = desc.size;
778780
std::vector<char> buffer(bytes_read);
781+
DBUG_EXECUTE_IF("TableReader.parse_deletion_vector.io_error", {
782+
create_status = Status::IOError("injected format v2 deletion vector read failure");
783+
return nullptr;
784+
});
785+
DBUG_EXECUTE_IF("TableReader.parse_deletion_vector.should_stop", {
786+
create_status = Status::EndOfFile("stop read.");
787+
return nullptr;
788+
});
779789
create_status = dv_reader.read_at(desc.start_offset, {buffer.data(), bytes_read});
780790
if (!create_status.ok()) [[unlikely]] {
781791
return nullptr;
782792
}
783793

784794
const char* buf = buffer.data();
785795
SCOPED_TIMER(_profile.parse_delete_file_time);
786-
create_status = parse_deletion_vector(buf, bytes_read, desc.format, delete_rows);
796+
create_status = parse_deletion_vector(buf, bytes_read, desc.format, delete_rows.get());
787797
if (!create_status.ok()) [[unlikely]] {
788798
return nullptr;
789799
}
790800
COUNTER_UPDATE(_profile.num_delete_rows, delete_rows->size());
791-
return delete_rows;
801+
return delete_rows.release();
792802
});
793803
RETURN_IF_ERROR(create_status);
794804
}

0 commit comments

Comments
 (0)