Skip to content

Commit 8f525f2

Browse files
authored
[fix](bufferedreader) Fix file size less than buffer (apache#1594)
When the file's size is less than whole buffer size, we should make the whole buffer size is equal to the file size.
1 parent ef29e86 commit 8f525f2

2 files changed

Lines changed: 23 additions & 17 deletions

File tree

‎be/src/io/buffered_reader.cpp‎

Lines changed: 21 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,8 @@ namespace doris {
3535
// add bvar to capture the download bytes per second by buffered reader
3636
bvar::Adder<uint64_t> g_bytes_downloaded("buffered_reader", "bytes_downloaded");
3737
bvar::PerSecond<bvar::Adder<uint64_t>> g_bytes_downloaded_per_second("buffered_reader",
38-
"bytes_downloaded_per_second",
39-
&g_bytes_downloaded, 60);
38+
"bytes_downloaded_per_second",
39+
&g_bytes_downloaded, 60);
4040

4141
// there exists occasions where the buffer is already closed but
4242
// some prior tasks are still queued in threadpool, so we have to check wheher
@@ -120,7 +120,8 @@ Status PrefetchBuffer::read_buffer(int64_t off, uint8_t* out, int64_t buf_len,
120120
int64_t read_len = std::min({buf_len, _offset + _size - off, _offset + _len - off});
121121
memcpy(out, _buf.data() + (off - _offset), read_len);
122122
*bytes_read = read_len;
123-
if (off + *bytes_read == _offset + _len) {
123+
auto new_off = _offset + _whole_buffer_size;
124+
if ((off + *bytes_read == _offset + _len) && new_off < _reader->size()) {
124125
reset_offset(_offset + _whole_buffer_size);
125126
}
126127
return Status::OK();
@@ -137,22 +138,10 @@ void PrefetchBuffer::close() {
137138
// buffered reader
138139
BufferedReader::BufferedReader(RuntimeProfile* profile, FileReader* reader, int64_t buffer_size)
139140
: _profile(profile), _reader(reader) {
140-
if (buffer_size == -1L) {
141+
if (buffer_size <= 0) {
141142
buffer_size = config::remote_storage_read_buffer_mb * 1024 * 1024;
142143
}
143144
_whole_pre_buffer_size = buffer_size;
144-
#ifdef BE_TEST
145-
s_max_pre_buffer_size = config::prefetch_single_buffer_size_mb;
146-
#endif
147-
int buffer_num = buffer_size > s_max_pre_buffer_size ? buffer_size / s_max_pre_buffer_size : 1;
148-
// set the _cur_offset of this reader as same as the inner reader's,
149-
// to make sure the buffer reader will start to read at right position.
150-
_reader->tell(&_cur_offset);
151-
for (int i = 0; i < buffer_num; i++) {
152-
_pre_buffers.emplace_back(std::make_shared<PrefetchBuffer>(
153-
profile, _cur_offset + i * s_max_pre_buffer_size, s_max_pre_buffer_size,
154-
_whole_pre_buffer_size, _reader.get()));
155-
}
156145
}
157146

158147
BufferedReader::~BufferedReader() {
@@ -179,6 +168,22 @@ Status BufferedReader::open() {
179168

180169
RETURN_IF_ERROR(_reader->open());
181170
_reader_size = _reader->size();
171+
172+
_whole_pre_buffer_size = std::min(_whole_pre_buffer_size, _reader->size());
173+
auto buffer_size = _whole_pre_buffer_size;
174+
#ifdef BE_TEST
175+
s_max_pre_buffer_size = config::prefetch_single_buffer_size_mb;
176+
#endif
177+
int64_t buffer_num = (buffer_size + s_max_pre_buffer_size - 1) / s_max_pre_buffer_size;
178+
// set the _cur_offset of this reader as same as the inner reader's,
179+
// to make sure the buffer reader will start to read at right position.
180+
_reader->tell(&_cur_offset);
181+
for (int i = 0; i < buffer_num; i++) {
182+
_pre_buffers.emplace_back(std::make_shared<PrefetchBuffer>(
183+
_profile, _cur_offset + i * s_max_pre_buffer_size, s_max_pre_buffer_size,
184+
_whole_pre_buffer_size, _reader.get()));
185+
}
186+
182187
// init all buffer and submit task
183188
resetAllBuffer(_cur_offset);
184189
return Status::OK();

‎be/src/io/file_factory.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,8 @@ doris::Status doris::FileFactory::create_file_reader(
8888
FileReader* hdfs_reader = nullptr;
8989
RETURN_IF_ERROR(HdfsReaderWriter::create_reader(range.hdfs_params, range.path, start_offset,
9090
&hdfs_reader));
91-
file_reader_ptr = new BufferedReader(profile, hdfs_reader);
91+
// temporarily hdfs reader is not thread safe, shouldn't use buffered reader
92+
file_reader_ptr = hdfs_reader;
9293
break;
9394
}
9495
default:

0 commit comments

Comments
 (0)