Skip to content

Commit bf58c2c

Browse files
authored
[fix](packed-file) Honor write_file_cache option for packed small files (apache#69693)
### What problem does this PR solve? Issue Number: None Related PR: apache#59892, apache#60411, apache#64995 Problem Summary: When packed file is enabled, the first segment file (`.dat`) and the index file (`.idx`) of a rowset are written through `PackedFileSystem`. The packed path decides whether to cache each small file from `PackedAppendContext::write_file_cache`, which defaults to `true`. `RowsetWriterContext` never sets this field.
1 parent 7865646 commit bf58c2c

4 files changed

Lines changed: 33 additions & 8 deletions

File tree

‎be/src/io/fs/packed_file_system.cpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
#include <string_view>
2222
#include <utility>
2323

24-
#include "cloud/config.h"
2524
#include "common/status.h"
2625
#include "io/fs/file_reader.h"
2726
#include "io/fs/packed_file_reader.h"
@@ -99,8 +98,10 @@ Status PackedFileSystem::create_file_impl(const Path& file, FileWriterPtr* write
9998
return Status::OK();
10099
}
101100

101+
// Follow the caller's file cache decision so that packed small files behave the same as
102+
// files written directly, e.g. a load with disable_file_cache=true must not populate cache.
102103
auto append_info = _append_info;
103-
if (config::enable_file_cache_write_index_file_only && opts != nullptr) {
104+
if (opts != nullptr) {
104105
append_info.write_file_cache = opts->write_file_cache;
105106
}
106107

‎be/src/io/fs/packed_file_writer.h‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -57,10 +57,6 @@ class PackedFileWriter final : public FileWriter {
5757
// Returns empty index if file is not in merge file
5858
Status get_packed_slice_location(PackedSliceLocation* location) const override;
5959

60-
#ifdef BE_TEST
61-
size_t buffer_capacity_for_test() const { return _buffer.capacity(); }
62-
#endif
63-
6460
private:
6561
void _release_buffer();
6662

‎be/test/io/fs/packed_file_system_test.cpp‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,34 @@ TEST_F(PackedFileSystemTest, NonzeroFirstSegmentUsesPackedWriter) {
324324
EXPECT_NE(dynamic_cast<PackedFileWriter*>(first_segment_index_writer.get()), nullptr);
325325
}
326326

327+
TEST_F(PackedFileSystemTest, CreateFileFollowsWriteFileCacheOption) {
328+
ASSERT_FALSE(config::enable_file_cache_write_index_file_only);
329+
PackedFileSystem merge_fs(_inner_fs, _append_info);
330+
331+
// A load with disable_file_cache=true must not write packed small files to file cache.
332+
FileWriterOptions no_cache_opts {.write_file_cache = false};
333+
FileWriterPtr no_cache_writer;
334+
ASSERT_TRUE(
335+
merge_fs.create_file(Path("rowset_1_0.dat"), &no_cache_writer, &no_cache_opts).ok());
336+
auto* packed_writer = dynamic_cast<PackedFileWriter*>(no_cache_writer.get());
337+
ASSERT_NE(packed_writer, nullptr);
338+
EXPECT_FALSE(packed_writer->_append_info.write_file_cache);
339+
340+
FileWriterOptions cache_opts {.write_file_cache = true};
341+
FileWriterPtr cache_writer;
342+
ASSERT_TRUE(merge_fs.create_file(Path("rowset_1_0.idx"), &cache_writer, &cache_opts).ok());
343+
packed_writer = dynamic_cast<PackedFileWriter*>(cache_writer.get());
344+
ASSERT_NE(packed_writer, nullptr);
345+
EXPECT_TRUE(packed_writer->_append_info.write_file_cache);
346+
347+
// Without options, keep the write_file_cache from append context.
348+
FileWriterPtr default_writer;
349+
ASSERT_TRUE(merge_fs.create_file(Path("rowset_1_0.dat"), &default_writer, nullptr).ok());
350+
packed_writer = dynamic_cast<PackedFileWriter*>(default_writer.get());
351+
ASSERT_NE(packed_writer, nullptr);
352+
EXPECT_TRUE(packed_writer->_append_info.write_file_cache);
353+
}
354+
327355
TEST_F(PackedFileSystemTest, OpenFileNotInMergeFile) {
328356
PackedFileSystem merge_fs(_inner_fs, _append_info);
329357

‎be/test/io/fs/packed_file_writer_test.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -181,14 +181,14 @@ TEST_F(PackedFileWriterTest, SwitchToDirectWriteReleasesBufferedMemory) {
181181
std::string small_data(80, 'a');
182182
Slice small_slice(small_data);
183183
ASSERT_TRUE(writer.appendv(&small_slice, 1).ok());
184-
EXPECT_GT(writer.buffer_capacity_for_test(), default_capacity);
184+
EXPECT_GT(writer._buffer.capacity(), default_capacity);
185185

186186
std::string large_data(30, 'b');
187187
Slice large_slice(large_data);
188188
ASSERT_TRUE(writer.appendv(&large_slice, 1).ok());
189189

190190
EXPECT_GT(inner_writer_ptr->append_calls(), 0);
191-
EXPECT_LE(writer.buffer_capacity_for_test(), default_capacity);
191+
EXPECT_LE(writer._buffer.capacity(), default_capacity);
192192
}
193193

194194
TEST_F(PackedFileWriterTest, CloseAsync) {

0 commit comments

Comments
 (0)