Skip to content

Commit 291f428

Browse files
committed
dynamic cast
1 parent 4d4bf5b commit 291f428

4 files changed

Lines changed: 30 additions & 2 deletions

File tree

‎be/src/io/fs/hdfs_file_writer.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,8 @@ void HdfsFileWriter::_write_into_local_file_cache() {
164164
size_t block_size = block->range().size();
165165
size_t append_size = std::min(data_remain_size, block_size);
166166
if (block->state() == FileBlock::State::EMPTY) {
167-
if (_index_offset != 0 && block->range().right >= _index_offset) {
167+
if (_index_offset != 0 && block->range().right >= _index_offset &&
168+
block->cache_type() == FileCacheType::NORMAL) {
168169
static_cast<void>(block->change_cache_type_self(FileCacheType::INDEX));
169170
}
170171
block->get_or_set_downloader();

‎be/src/io/fs/hdfs_file_writer.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,12 @@ class HdfsFileWriter final : public FileWriter {
5050
size_t bytes_appended() const override { return _bytes_appended; }
5151
bool closed() const override { return _closed; }
5252

53+
void mark_index_offset() {
54+
if (_cache_builder != nullptr) {
55+
_index_offset = _bytes_appended;
56+
}
57+
}
58+
5359
private:
5460
// Flush buffered data into HDFS client and write local file cache if enabled
5561
// **Notice**: this would clear the underlying buffer

‎be/src/io/fs/s3_file_writer.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,17 @@ class S3FileWriter final : public FileWriter {
5858
size_t bytes_appended() const override { return _bytes_appended; }
5959
bool closed() const override { return _closed; }
6060

61+
void mark_index_offset() {
62+
if (_expiration_time == 0) {
63+
_index_offset = _bytes_appended;
64+
// Only the normal data need to change to index data
65+
if (_pending_buf) {
66+
std::dynamic_pointer_cast<UploadFileBuffer>(_pending_buf)
67+
->set_index_offset(_index_offset);
68+
}
69+
}
70+
}
71+
6172
private:
6273
Status _abort();
6374
[[nodiscard]] std::string _dump_completed_part() const;

‎be/src/olap/rowset/segment_v2/segment_writer.cpp‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@
3737
#include "io/fs/file_system.h"
3838
#include "io/fs/file_writer.h"
3939
#include "io/fs/local_file_system.h"
40+
#include "io/fs/hdfs_file_writer.h"
41+
#include "io/fs/s3_file_writer.h"
4042
#include "olap/data_dir.h"
4143
#include "olap/key_coder.h"
4244
#include "olap/olap_common.h"
@@ -196,7 +198,7 @@ Status SegmentWriter::init(const std::vector<uint32_t>& col_ids, bool has_key) {
196198
if (_opts.compression_type == UNKNOWN_COMPRESSION) {
197199
_opts.compression_type = _tablet_schema->compression_type();
198200
}
199-
auto create_column_writer = [&](uint32_t cid, const auto& column) -> auto {
201+
auto create_column_writer = [&](uint32_t cid, const auto& column) -> auto{
200202
ColumnWriterOptions opts;
201203
opts.meta = _footer.add_columns();
202204

@@ -1056,6 +1058,14 @@ Status SegmentWriter::finalize_columns_data() {
10561058
}
10571059

10581060
Status SegmentWriter::finalize_columns_index(uint64_t* index_size) {
1061+
if (config::is_cloud_mode()) {
1062+
if (auto* s3_writer = dynamic_cast<io::S3FileWriter*>(_file_writer); s3_writer) {
1063+
s3_writer->mark_index_offset();
1064+
}
1065+
if (auto* hdfs_writer = dynamic_cast<io::HdfsFileWriter*>(_file_writer); hdfs_writer) {
1066+
hdfs_writer->mark_index_offset();
1067+
}
1068+
}
10591069
uint64_t index_start = _file_writer->bytes_appended();
10601070
RETURN_IF_ERROR(_write_ordinal_index());
10611071
RETURN_IF_ERROR(_write_zone_map());

0 commit comments

Comments
 (0)