Skip to content

Commit 303e760

Browse files
committed
[fix](be) Forward Parquet profile cache through load reader
1 parent c80a4a5 commit 303e760

2 files changed

Lines changed: 16 additions & 2 deletions

File tree

‎be/src/format_v2/table/load_reader.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ Status LoadTableReader::init(TableReadOptions&& options) {
8888
.io_ctx = options.io_ctx,
8989
.runtime_state = options.runtime_state,
9090
.scanner_profile = options.scanner_profile,
91+
.parquet_profile_cache = options.parquet_profile_cache,
9192
.file_slot_descs = &_source_file_slots,
9293
.push_down_agg_type = TPushAggOp::NONE,
9394
.condition_cache_digest = 0});

‎be/test/format_v2/table/load_reader_test.cpp‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
#include "core/data_type/data_type_nullable.h"
3131
#include "core/data_type/data_type_number.h"
3232
#include "core/data_type/data_type_string.h"
33+
#include "format_v2/parquet/parquet_profile.h"
3334
#include "io/fs/kafka_consumer_pipe.h"
3435
#include "load/stream_load/new_load_stream_mgr.h"
3536
#include "load/stream_load/stream_load_context.h"
@@ -130,6 +131,7 @@ class TestLoadTableReader final : public LoadTableReader {
130131
TestLoadTableReader(std::unique_ptr<SourceReader> source_reader, TupleDescriptor* source,
131132
TupleDescriptor* destination)
132133
: LoadTableReader(source, destination), _input(std::move(source_reader)) {}
134+
parquet::ParquetProfileCache* parquet_profile_cache() const { return _parquet_profile_cache; }
133135
Status init(TableReadOptions&& options) override {
134136
RETURN_IF_ERROR(LoadTableReader::init(std::move(options)));
135137
_input->columns = _projected_columns;
@@ -236,7 +238,8 @@ class LoadReaderTest : public testing::Test {
236238
cast.__set_num_children(1);
237239
params.expr_of_dest_slot[20].nodes.insert(params.expr_of_dest_slot[20].nodes.begin(), cast);
238240
}
239-
Status init(LoadTableReader& reader, FileFormat format = FileFormat::CSV) {
241+
Status init(LoadTableReader& reader, FileFormat format = FileFormat::CSV,
242+
parquet::ParquetProfileCache* parquet_profile_cache = nullptr) {
240243
TTupleDescriptor tuple;
241244
tuple.__set_id(0);
242245
tuple.__set_byteSize(0);
@@ -259,7 +262,8 @@ class LoadReaderTest : public testing::Test {
259262
.scan_params = &params,
260263
.io_ctx = io,
261264
.runtime_state = &state,
262-
.scanner_profile = &profile});
265+
.scanner_profile = &profile,
266+
.parquet_profile_cache = parquet_profile_cache});
263267
}
264268
void add_partial_update_columns(bool key) {
265269
TTupleDescriptor tuple;
@@ -340,6 +344,15 @@ class LoadReaderTest : public testing::Test {
340344
std::unique_ptr<SlotDescriptor> source_slot, destination_slot;
341345
};
342346

347+
TEST_F(LoadReaderTest, ForwardsParquetProfileCacheToBaseReader) {
348+
parquet::ParquetProfileCache cache(&profile);
349+
TestLoadTableReader reader(std::make_unique<SourceReader>(), source.get(), destination.get());
350+
const auto status = init(reader, FileFormat::PARQUET, &cache);
351+
ASSERT_TRUE(status.ok()) << status;
352+
EXPECT_EQ(reader.parquet_profile_cache(), &cache);
353+
ASSERT_TRUE(reader.close().ok());
354+
}
355+
343356
class LoadJsonPipeTest : public LoadReaderTest {
344357
protected:
345358
void SetUp() override {

0 commit comments

Comments
 (0)