Skip to content

Commit 70bf96e

Browse files
authored
[feat](iceberg) HDFS lazy open + iceberg delete file file_size propagation (#66773) (#67842)
Cherry-picked from #66773 ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test <!-- At least one of them must be included. --> - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason <!-- Add your reason? --> - Behavior changed: - [ ] No. - [ ] Yes. <!-- Explain the behavior change --> - Does this need documentation? - [ ] No. - [ ] Yes. <!-- Add document PR link here. eg: apache/doris-website#1214 --> ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into -->
1 parent 97bc869 commit 70bf96e

28 files changed

Lines changed: 1122 additions & 65 deletions

‎be/src/exec/scan/file_scanner.cpp‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -575,8 +575,14 @@ Status FileScanner::_get_block_wrapped(RuntimeState* state, Block* block, bool*
575575

576576
// Read next block.
577577
// Some of column in block may not be filled (column not exist in file)
578-
RETURN_IF_ERROR(
579-
_cur_reader->get_next_block(_src_block_ptr, &read_rows, &_cur_reader_eof));
578+
Status st = _cur_reader->get_next_block(_src_block_ptr, &read_rows, &_cur_reader_eof);
579+
// Lazy open may surface NOT_FOUND on the first read; skip as above.
580+
if (st.is<ErrorCode::NOT_FOUND>() && config::ignore_not_found_file_in_external_table) {
581+
_cur_reader_eof = true;
582+
COUNTER_UPDATE(_not_found_file_counter, 1);
583+
continue;
584+
}
585+
RETURN_IF_ERROR(st);
580586
}
581587
// use read_rows instead of _src_block_ptr->rows(), because the first column of _src_block_ptr
582588
// may not be filled after calling `get_next_block()`, so _src_block_ptr->rows() may return wrong result.

‎be/src/format/table/iceberg_delete_file_reader_helper.cpp‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -243,12 +243,12 @@ TFileScanRangeParams build_iceberg_delete_scan_range_params(
243243
return params;
244244
}
245245

246-
TFileRangeDesc build_iceberg_delete_file_range(const std::string& path) {
246+
TFileRangeDesc build_iceberg_delete_file_range(const std::string& path, int64_t file_size) {
247247
TFileRangeDesc range;
248248
range.path = path;
249249
range.start_offset = 0;
250250
range.size = -1;
251-
range.file_size = -1;
251+
range.file_size = file_size;
252252
return range;
253253
}
254254

@@ -283,7 +283,8 @@ Status read_iceberg_position_delete_file(const TIcebergDeleteFileDesc& delete_fi
283283
return Status::InvalidArgument("invalid position delete reader options");
284284
}
285285

286-
TFileRangeDesc delete_range = build_iceberg_delete_file_range(delete_file.path);
286+
TFileRangeDesc delete_range = build_iceberg_delete_file_range(
287+
delete_file.path, delete_file.__isset.file_size ? delete_file.file_size : -1);
287288
if (options.fs_name != nullptr && !options.fs_name->empty()) {
288289
delete_range.__set_fs_name(*options.fs_name);
289290
}
@@ -362,7 +363,8 @@ Status read_iceberg_deletion_vector(const TIcebergDeleteFileDesc& delete_file,
362363
DBUG_EXECUTE_IF("IcebergDeleteFileReader.read_deletion_vector.should_stop",
363364
{ return Status::EndOfFile("stop read."); });
364365

365-
TFileRangeDesc delete_range = build_iceberg_delete_file_range(delete_file.path);
366+
TFileRangeDesc delete_range = build_iceberg_delete_file_range(
367+
delete_file.path, delete_file.__isset.file_size ? delete_file.file_size : -1);
366368
if (options.fs_name != nullptr && !options.fs_name->empty()) {
367369
delete_range.__set_fs_name(*options.fs_name);
368370
}

‎be/src/format/table/iceberg_delete_file_reader_helper.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ TFileScanRangeParams build_iceberg_delete_scan_range_params(
6767
const std::map<std::string, std::string>& hadoop_conf, TFileType::type file_type,
6868
const std::vector<TNetworkAddress>& broker_addresses);
6969

70-
TFileRangeDesc build_iceberg_delete_file_range(const std::string& path);
70+
TFileRangeDesc build_iceberg_delete_file_range(const std::string& path, int64_t file_size);
7171

7272
bool is_iceberg_deletion_vector(const TIcebergDeleteFileDesc& delete_file);
7373

‎be/src/format/table/iceberg_reader.cpp‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1415,14 +1415,14 @@ Status IcebergTableReader::_position_delete_base(
14151415
Status create_status = Status::OK();
14161416
auto* delete_file_cache = _kv_cache->get<DeleteFile>(
14171417
_delet_file_cache_key(delete_file.path), [&]() -> DeleteFile* {
1418-
auto* position_delete = new DeleteFile;
1419-
create_status = _read_position_delete_file(delete_file, position_delete);
1418+
auto position_delete = std::make_unique<DeleteFile>();
1419+
create_status = _read_position_delete_file(delete_file, position_delete.get());
14201420

14211421
if (!create_status) {
14221422
return nullptr;
14231423
}
14241424

1425-
return position_delete;
1425+
return position_delete.release();
14261426
});
14271427
if (create_status.is<ErrorCode::END_OF_FILE>()) {
14281428
continue;
@@ -1463,6 +1463,17 @@ Status IcebergTableReader::_position_delete_base(
14631463
return Status::OK();
14641464
}
14651465

1466+
Status IcebergTableReader::TEST_position_delete_base(
1467+
const std::string& data_file_path,
1468+
const std::vector<TIcebergDeleteFileDesc>& delete_files) {
1469+
return _position_delete_base(data_file_path, delete_files);
1470+
}
1471+
1472+
Status IcebergTableReader::TEST_read_equality_delete_file(
1473+
const TIcebergDeleteFileDesc& delete_file) {
1474+
return _process_equality_delete({delete_file});
1475+
}
1476+
14661477
Status IcebergTableReader::_read_position_delete_file(const TIcebergDeleteFileDesc& delete_file,
14671478
DeleteFile* position_delete) {
14681479
GroupedDeleteRowsVisitor visitor(position_delete);
@@ -2060,7 +2071,8 @@ Status IcebergTableReader::read_deletion_vector(const std::string& data_file_pat
20602071
delete_range.path = delete_file_desc.path;
20612072
delete_range.start_offset = delete_file_desc.content_offset;
20622073
delete_range.size = delete_file_desc.content_size_in_bytes;
2063-
delete_range.file_size = -1;
2074+
delete_range.file_size =
2075+
delete_file_desc.__isset.file_size ? delete_file_desc.file_size : -1;
20642076

20652077
// We may consider caching the DeletionVectorReader when reading Puffin files,
20662078
// where the underlying reader is an `InMemoryFileReader` and a single data file is
@@ -2155,7 +2167,7 @@ Status IcebergParquetReader::_process_equality_delete(
21552167
delete_desc.path = delete_file.path;
21562168
delete_desc.start_offset = 0;
21572169
delete_desc.size = -1;
2158-
delete_desc.file_size = -1;
2170+
delete_desc.file_size = delete_file.__isset.file_size ? delete_file.file_size : -1;
21592171

21602172
auto delete_reader = ParquetReader::create_unique(
21612173
_profile, _params, delete_desc, READ_DELETE_FILE_BATCH_SIZE,
@@ -2302,7 +2314,7 @@ Status IcebergOrcReader::_process_equality_delete(
23022314
delete_desc.path = delete_file.path;
23032315
delete_desc.start_offset = 0;
23042316
delete_desc.size = -1;
2305-
delete_desc.file_size = -1;
2317+
delete_desc.file_size = delete_file.__isset.file_size ? delete_file.file_size : -1;
23062318

23072319
auto delete_reader = OrcReader::create_unique(_profile, _state, _params, delete_desc,
23082320
READ_DELETE_FILE_BATCH_SIZE,

‎be/src/format/table/iceberg_reader.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,10 @@ class IcebergTableReader : public TableFormatReader, public TableSchemaChangeHel
100100
enum { DATA, POSITION_DELETE, EQUALITY_DELETE, DELETION_VECTOR };
101101
enum Fileformat { NONE, PARQUET, ORC, AVRO };
102102

103+
Status TEST_position_delete_base(const std::string& data_file_path,
104+
const std::vector<TIcebergDeleteFileDesc>& delete_files);
105+
Status TEST_read_equality_delete_file(const TIcebergDeleteFileDesc& delete_file);
106+
103107
virtual void set_delete_rows() = 0;
104108

105109
Status read_deletion_vector(const std::string& data_file_path,

‎be/src/format_v2/orc/orc_reader.cpp‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -930,8 +930,15 @@ Status OrcReader::init(RuntimeState* state) {
930930
if (is_orc_stop(_io_ctx.get(), e)) {
931931
return Status::EndOfFile("stop");
932932
}
933+
// invoker maybe just skip Status.NotFound and continue
934+
// so we need distinguish between it and other kinds of errors
935+
const std::string err_msg = e.what();
936+
if (err_msg.find("No such file or directory") != std::string::npos ||
937+
err_msg.find("NoSuchKey") != std::string::npos) {
938+
return Status::NotFound(err_msg);
939+
}
933940
return Status::InternalError("Failed to open ORC file {}: {}", _file_description->path,
934-
e.what());
941+
err_msg);
935942
}
936943
return Status::OK();
937944
};

‎be/src/format_v2/table/iceberg_reader.cpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1144,7 +1144,7 @@ Status IcebergTableReader::_parse_deletion_vector_file(const TTableFormatFileDes
11441144
desc->path = deletion_vector->path;
11451145
desc->start_offset = deletion_vector->content_offset;
11461146
desc->size = static_cast<int64_t>(bytes_read);
1147-
desc->file_size = -1;
1147+
desc->file_size = deletion_vector->__isset.file_size ? deletion_vector->file_size : -1;
11481148
desc->format = DeleteFileDesc::Format::ICEBERG;
11491149
*has_delete_file = true;
11501150
return Status::OK();
@@ -1500,7 +1500,8 @@ Status IcebergTableReader::_create_delete_file_reader(const TIcebergDeleteFileDe
15001500
return Status::NotSupported("Unsupported Iceberg delete file format {}",
15011501
delete_file.file_format);
15021502
}
1503-
auto delete_range = build_iceberg_delete_file_range(delete_file.path);
1503+
auto delete_range = build_iceberg_delete_file_range(
1504+
delete_file.path, delete_file.__isset.file_size ? delete_file.file_size : -1);
15041505
if (_current_task != nullptr && _current_task->data_file != nullptr &&
15051506
!_current_task->data_file->fs_name.empty()) {
15061507
delete_range.__set_fs_name(_current_task->data_file->fs_name);

‎be/src/io/fs/file_handle_cache.cpp‎

Lines changed: 53 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,11 @@
2626
#include <tuple>
2727

2828
#include "common/cast_set.h"
29+
#include "common/metrics/doris_metrics.h"
30+
#include "cpp/sync_point.h"
2931
#include "io/fs/err_utils.h"
32+
#include "io/hdfs_util.h"
33+
#include "util/bvar_helper.h"
3034
#include "util/hash_util.hpp"
3135
#include "util/time.h"
3236
namespace doris::io {
@@ -36,36 +40,62 @@ namespace doris::io {
3640
HdfsFileHandle::~HdfsFileHandle() {
3741
if (_hdfs_file != nullptr && _fs != nullptr) {
3842
VLOG_FILE << "hdfsCloseFile() fid=" << _hdfs_file;
39-
hdfsCloseFile(_fs, _hdfs_file); // TODO: check return code
43+
SCOPED_BVAR_LATENCY(hdfs_bvar::hdfs_close_latency);
44+
SYNC_POINT_HOOK_RETURN_VALUE(hdfsCloseFile(_fs, _hdfs_file),
45+
"HdfsFileHandle::close::hdfsCloseFile");
46+
DorisMetrics::instance()->hdfs_file_open_reading->increment(-1);
4047
}
4148
_fs = nullptr;
4249
_hdfs_file = nullptr;
4350
}
4451

4552
Status HdfsFileHandle::init(int64_t file_size) {
46-
_hdfs_file = hdfsOpenFile(_fs, _fname.c_str(), O_RDONLY, 0, 0, 0);
47-
if (_hdfs_file == nullptr) {
48-
std::string _err_msg = hdfs_error();
49-
// invoker maybe just skip Status.NotFound and continue
50-
// so we need distinguish between it and other kinds of errors
51-
if (_err_msg.find("No such file or directory") != std::string::npos) {
52-
return Status::NotFound(_err_msg);
53-
}
54-
return Status::InternalError("failed to open {}: {}", _fname, _err_msg);
55-
}
56-
5753
_file_size = file_size;
5854
if (_file_size <= 0) {
59-
hdfsFileInfo* file_info = hdfsGetPathInfo(_fs, _fname.c_str());
55+
SCOPED_BVAR_LATENCY(hdfs_bvar::hdfs_get_path_info_latency);
56+
auto* file_info = SYNC_POINT_HOOK_RETURN_VALUE(hdfsGetPathInfo(_fs, _fname.c_str()),
57+
"HdfsFileHandle::init::hdfsGetPathInfo");
6058
if (file_info == nullptr) {
61-
return Status::InternalError("failed to get file size of {}: {}", _fname, hdfs_error());
59+
std::string err_msg =
60+
SYNC_POINT_HOOK_RETURN_VALUE(hdfs_error(), "HdfsFileHandle::init::hdfs_error");
61+
// invoker maybe just skip Status.NotFound and continue
62+
// so we need distinguish between it and other kinds of errors
63+
if (err_msg.find("No such file or directory") != std::string::npos) {
64+
return Status::NotFound(err_msg);
65+
}
66+
return Status::InternalError("failed to get file size of {}: {}", _fname, err_msg);
6267
}
6368
_file_size = file_info->mSize;
6469
hdfsFreeFileInfo(file_info, 1);
6570
}
6671
return Status::OK();
6772
}
6873

74+
Status HdfsFileHandle::ensure_open() {
75+
std::call_once(_open_once, [this]() {
76+
VLOG_DEBUG << "lazy open hdfs file: " << _fname;
77+
SCOPED_BVAR_LATENCY(hdfs_bvar::hdfs_open_latency);
78+
_hdfs_file =
79+
SYNC_POINT_HOOK_RETURN_VALUE(hdfsOpenFile(_fs, _fname.c_str(), O_RDONLY, 0, 0, 0),
80+
"HdfsFileHandle::ensure_open::hdfsOpenFile");
81+
if (_hdfs_file != nullptr) {
82+
_open_status = Status::OK();
83+
DorisMetrics::instance()->hdfs_file_open_reading->increment(1);
84+
DorisMetrics::instance()->hdfs_file_reader_total->increment(1);
85+
} else {
86+
// Capture error inside the opening thread (libhdfs last-error is thread-local).
87+
std::string _err_msg = SYNC_POINT_HOOK_RETURN_VALUE(
88+
hdfs_error(), "HdfsFileHandle::ensure_open::hdfs_error");
89+
if (_err_msg.find("No such file or directory") != std::string::npos) {
90+
_open_status = Status::NotFound(_err_msg);
91+
} else {
92+
_open_status = Status::InternalError("failed to open {}: {}", _fname, _err_msg);
93+
}
94+
}
95+
});
96+
return _open_status;
97+
}
98+
6999
CachedHdfsFileHandle::CachedHdfsFileHandle(const hdfsFS& fs, const std::string& fname,
70100
int64_t mtime)
71101
: HdfsFileHandle(fs, fname, mtime) {}
@@ -100,8 +130,16 @@ void FileHandleCache::Accessor::destroy() {
100130

101131
FileHandleCache::Accessor::~Accessor() {
102132
if (_cache_accessor.get()) {
133+
auto* handle = get();
134+
if (handle->file() == nullptr) {
135+
// Not opened or open failed (call_once won't retry), destroy to avoid cache pollution
136+
destroy();
137+
return;
138+
}
103139
#ifdef USE_HADOOP_HDFS
104-
if (hdfsUnbufferFile(get()->file()) != 0) {
140+
int unbuffer_ret = SYNC_POINT_HOOK_RETURN_VALUE(hdfsUnbufferFile(handle->file()),
141+
"HdfsFileHandle::close::hdfsUnbufferFile");
142+
if (unbuffer_ret != 0) {
105143
VLOG_FILE << "FS does not support file handle unbuffering, closing file="
106144
<< _cache_accessor.get_key()->second.first;
107145
destroy();

‎be/src/io/fs/file_handle_cache.h‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include <list>
2727
#include <map>
2828
#include <memory>
29+
#include <mutex>
2930
#include <string>
3031
#include <utility>
3132

@@ -49,9 +50,12 @@ class HdfsFileHandle {
4950
/// Destructor will close the file handle
5051
~HdfsFileHandle();
5152

52-
/// Init opens the file handle
53+
/// Init only sets file_size (from param or hdfsGetPathInfo), does NOT open the file.
5354
Status init(int64_t file_size);
5455

56+
/// Lazily opens the file handle on first read. Thread-safe via std::call_once.
57+
Status ensure_open();
58+
5559
hdfsFS fs() const { return _fs; }
5660
hdfsFile file() const { return _hdfs_file; }
5761
int64_t mtime() const { return _mtime; }
@@ -66,7 +70,9 @@ class HdfsFileHandle {
6670
const std::string _fname;
6771
hdfsFile _hdfs_file = nullptr;
6872
int64_t _mtime;
69-
int64_t _file_size;
73+
int64_t _file_size = -1;
74+
std::once_flag _open_once;
75+
Status _open_status;
7076
};
7177

7278
/// CachedHdfsFileHandles are owned by the file handle cache and are used for no

0 commit comments

Comments
 (0)