@@ -35,8 +35,8 @@ namespace doris {
3535// add bvar to capture the download bytes per second by buffered reader
3636bvar::Adder<uint64_t > g_bytes_downloaded (" buffered_reader" , " bytes_downloaded" );
3737bvar::PerSecond<bvar::Adder<uint64_t >> g_bytes_downloaded_per_second (" buffered_reader" ,
38- " bytes_downloaded_per_second" ,
39- &g_bytes_downloaded, 60 );
38+ " bytes_downloaded_per_second" ,
39+ &g_bytes_downloaded, 60 );
4040
4141// there exists occasions where the buffer is already closed but
4242// some prior tasks are still queued in threadpool, so we have to check wheher
@@ -120,7 +120,8 @@ Status PrefetchBuffer::read_buffer(int64_t off, uint8_t* out, int64_t buf_len,
120120 int64_t read_len = std::min ({buf_len, _offset + _size - off, _offset + _len - off});
121121 memcpy (out, _buf.data () + (off - _offset), read_len);
122122 *bytes_read = read_len;
123- if (off + *bytes_read == _offset + _len) {
123+ auto new_off = _offset + _whole_buffer_size;
124+ if ((off + *bytes_read == _offset + _len) && new_off < _reader->size ()) {
124125 reset_offset (_offset + _whole_buffer_size);
125126 }
126127 return Status::OK ();
@@ -137,22 +138,10 @@ void PrefetchBuffer::close() {
137138// buffered reader
138139BufferedReader::BufferedReader (RuntimeProfile* profile, FileReader* reader, int64_t buffer_size)
139140 : _profile(profile), _reader(reader) {
140- if (buffer_size == - 1L ) {
141+ if (buffer_size <= 0 ) {
141142 buffer_size = config::remote_storage_read_buffer_mb * 1024 * 1024 ;
142143 }
143144 _whole_pre_buffer_size = buffer_size;
144- #ifdef BE_TEST
145- s_max_pre_buffer_size = config::prefetch_single_buffer_size_mb;
146- #endif
147- int buffer_num = buffer_size > s_max_pre_buffer_size ? buffer_size / s_max_pre_buffer_size : 1 ;
148- // set the _cur_offset of this reader as same as the inner reader's,
149- // to make sure the buffer reader will start to read at right position.
150- _reader->tell (&_cur_offset);
151- for (int i = 0 ; i < buffer_num; i++) {
152- _pre_buffers.emplace_back (std::make_shared<PrefetchBuffer>(
153- profile, _cur_offset + i * s_max_pre_buffer_size, s_max_pre_buffer_size,
154- _whole_pre_buffer_size, _reader.get ()));
155- }
156145}
157146
158147BufferedReader::~BufferedReader () {
@@ -179,6 +168,22 @@ Status BufferedReader::open() {
179168
180169 RETURN_IF_ERROR (_reader->open ());
181170 _reader_size = _reader->size ();
171+
172+ _whole_pre_buffer_size = std::min (_whole_pre_buffer_size, _reader->size ());
173+ auto buffer_size = _whole_pre_buffer_size;
174+ #ifdef BE_TEST
175+ s_max_pre_buffer_size = config::prefetch_single_buffer_size_mb;
176+ #endif
177+ int64_t buffer_num = (buffer_size + s_max_pre_buffer_size - 1 ) / s_max_pre_buffer_size;
178+ // set the _cur_offset of this reader as same as the inner reader's,
179+ // to make sure the buffer reader will start to read at right position.
180+ _reader->tell (&_cur_offset);
181+ for (int i = 0 ; i < buffer_num; i++) {
182+ _pre_buffers.emplace_back (std::make_shared<PrefetchBuffer>(
183+ _profile, _cur_offset + i * s_max_pre_buffer_size, s_max_pre_buffer_size,
184+ _whole_pre_buffer_size, _reader.get ()));
185+ }
186+
182187 // init all buffer and submit task
183188 resetAllBuffer (_cur_offset);
184189 return Status::OK ();
0 commit comments