Skip to content

Commit 3b20906

Browse files
committed
[feature](be) Add session controls for Parquet page indexes
### What problem does this PR solve? Problem Summary: Parquet output currently relies on implicit page-index and data-page-size defaults. Add validated session variables for controlling ColumnIndex/OffsetIndex output and the target data-page size across OUTFILE, EXPORT, Hive and Iceberg writes. Preserve the existing enabled index default, 1 MiB page target and Hadoop-framed LZ4 compatibility limit. ### Release note Add enable_parquet_write_page_index (default true) and parquet_write_data_page_size (default 1048576 bytes) for new Parquet output. ### Check List (For Author) - Test: 16 BE ASAN tests and 9 FE tests passed; clang-format 16, build hygiene, FE Checkstyle and clang-tidy passed. - Behavior changed: Yes; users can explicitly disable page indexes and adjust the target data-page size. Existing defaults are preserved. - Does this need documentation: Yes; paired English and Chinese Parquet documentation.
1 parent 5af2df3 commit 3b20906

5 files changed

Lines changed: 337 additions & 1 deletion

File tree

‎be/src/format/transformer/vparquet_writer.cpp‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#include <parquet/type_fwd.h>
2828
#include <parquet/types.h>
2929

30+
#include <algorithm>
3031
#include <ctime>
3132
#include <exception>
3233
#include <ostream>
@@ -46,6 +47,9 @@
4647
#include "util/timezone_utils.h"
4748

4849
namespace doris {
50+
namespace {
51+
constexpr int64_t HADOOP_LZ4_DEFAULT_BUFFER_SIZE = 256 * 1024;
52+
} // namespace
4953

5054
ParquetOutputStream::ParquetOutputStream(doris::io::FileWriter* file_writer)
5155
: _file_writer(file_writer), _cur_pos(0), _written_len(0) {
@@ -141,7 +145,6 @@ void ParquetBuildHelper::build_compression_type(
141145
break;
142146
}
143147
case TParquetCompressionType::LZ4_HADOOP: {
144-
constexpr int64_t HADOOP_LZ4_DEFAULT_BUFFER_SIZE = 256 * 1024;
145148
// Hadoop-framed LZ4 -> Parquet thrift codec "LZ4" (deprecated). This matches what
146149
// Spark/Iceberg writes for `write.parquet.compression-codec=lz4`. Arrow 17 emits one
147150
// Hadoop LZ4 block per Parquet page/dictionary page, while Hadoop JVM readers default to
@@ -213,6 +216,20 @@ Status VParquetWriter::_parse_properties() {
213216
::parquet::WriterProperties::Builder builder;
214217
ParquetBuildHelper::build_compression_type(builder, _parquet_options.compression_type);
215218
ParquetBuildHelper::build_version(builder, _parquet_options.parquet_version);
219+
const auto& query_options = _state->query_options();
220+
// Preserve the current enabled default explicitly, including when older FEs omit the field.
221+
if (query_options.enable_parquet_write_page_index) {
222+
builder.enable_write_page_index();
223+
} else {
224+
builder.disable_write_page_index();
225+
}
226+
int64_t page_size = query_options.parquet_write_data_page_size;
227+
DORIS_CHECK_GT(page_size, 0);
228+
// A 1 MiB session target must still respect the 128 KiB Hadoop-framed LZ4 limit.
229+
if (_parquet_options.compression_type == TParquetCompressionType::LZ4_HADOOP) {
230+
page_size = std::min(page_size, HADOOP_LZ4_DEFAULT_BUFFER_SIZE / 2);
231+
}
232+
builder.data_pagesize(page_size);
216233
if (_parquet_options.parquet_disable_dictionary) {
217234
builder.disable_dictionary();
218235
} else {

‎be/test/format/transformer/vparquet_writer_test.cpp‎

Lines changed: 226 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include <gtest/gtest.h>
2323
#include <parquet/api/reader.h>
2424
#include <parquet/arrow/reader.h>
25+
#include <parquet/page_index.h>
2526
#include <parquet/schema.h>
2627

2728
#include <algorithm>
@@ -178,11 +179,236 @@ class VParquetWriterTest : public testing::Test {
178179
EXPECT_EQ(offsets, repeated);
179180
}
180181

182+
Block page_index_block() {
183+
auto type = std::make_shared<DataTypeInt32>();
184+
auto nullable = make_nullable(type);
185+
auto ids = type->create_column();
186+
auto values = nullable->create_column();
187+
auto nulls = nullable->create_column();
188+
for (int32_t row = 0; row < 12000; ++row) {
189+
ids->insert(Field::create_field<TYPE_INT>(row));
190+
if (row % 7 == 0) {
191+
values->insert_default();
192+
} else {
193+
values->insert(Field::create_field<TYPE_INT>(row % 64));
194+
}
195+
nulls->insert_default();
196+
}
197+
return Block({{std::move(ids), type, "id"},
198+
{std::move(values), nullable, "value"},
199+
{std::move(nulls), nullable, "all_null"}});
200+
}
201+
202+
void check_page_indexes(bool enabled) {
203+
auto reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false);
204+
ASSERT_EQ(reader->metadata()->num_row_groups(), 1);
205+
auto group = reader->metadata()->RowGroup(0);
206+
ASSERT_EQ(group->num_rows(), 12000);
207+
for (int column = 0; column < group->num_columns(); ++column) {
208+
auto chunk = group->ColumnChunk(column);
209+
EXPECT_EQ(chunk->GetColumnIndexLocation().has_value(), enabled);
210+
EXPECT_EQ(chunk->GetOffsetIndexLocation().has_value(), enabled);
211+
if (!enabled) {
212+
continue;
213+
}
214+
auto index_reader = reader->GetPageIndexReader();
215+
ASSERT_NE(index_reader, nullptr);
216+
auto indexes = index_reader->RowGroup(0);
217+
ASSERT_NE(indexes, nullptr);
218+
auto column_index = indexes->GetColumnIndex(column);
219+
auto offset_index = indexes->GetOffsetIndex(column);
220+
ASSERT_NE(column_index, nullptr);
221+
ASSERT_NE(offset_index, nullptr);
222+
const auto& pages = offset_index->page_locations();
223+
ASSERT_FALSE(pages.empty());
224+
EXPECT_EQ(pages.size(), column_index->null_pages().size());
225+
EXPECT_EQ(pages.front().offset, chunk->data_page_offset());
226+
EXPECT_EQ(pages.front().first_row_index, 0);
227+
const auto start = chunk->has_dictionary_page() ? chunk->dictionary_page_offset()
228+
: chunk->data_page_offset();
229+
int64_t previous_row = -1;
230+
int64_t previous_end = start;
231+
for (const auto& page : pages) {
232+
EXPECT_GT(page.first_row_index, previous_row);
233+
EXPECT_LT(page.first_row_index, group->num_rows());
234+
EXPECT_GE(page.offset, previous_end);
235+
EXPECT_GT(page.compressed_page_size, 0);
236+
EXPECT_LE(page.offset + page.compressed_page_size,
237+
start + chunk->total_compressed_size());
238+
previous_row = page.first_row_index;
239+
previous_end = page.offset + page.compressed_page_size;
240+
}
241+
if (column == 0) {
242+
EXPECT_GT(pages.size(), 1);
243+
} else if (column == 2) {
244+
EXPECT_TRUE(std::all_of(column_index->null_pages().begin(),
245+
column_index->null_pages().end(),
246+
[](bool value) { return value; }));
247+
}
248+
}
249+
auto input = arrow::io::ReadableFile::Open(_file_path);
250+
ASSERT_TRUE(input.ok()) << input.status();
251+
auto arrow_reader = ::parquet::arrow::OpenFile(*input, arrow::default_memory_pool());
252+
ASSERT_TRUE(arrow_reader.ok()) << arrow_reader.status();
253+
auto table = (*arrow_reader)->ReadTable();
254+
ASSERT_TRUE(table.ok()) << table.status();
255+
EXPECT_EQ((*table)->num_rows(), 12000);
256+
EXPECT_EQ((*table)->column(1)->null_count(), (12000 + 6) / 7);
257+
EXPECT_EQ((*table)->column(2)->null_count(), 12000);
258+
const auto& ids = assert_cast<const arrow::Int32Array&>(*(*table)->column(0)->chunk(0));
259+
EXPECT_EQ(ids.Value(0), 0);
260+
EXPECT_EQ(ids.Value(11999), 11999);
261+
}
262+
181263
arrow::MemoryPool* _previous_pool = nullptr;
182264
std::string _file_path;
183265
std::shared_ptr<io::FileSystem> _fs;
184266
};
185267

268+
TEST_F(VParquetWriterTest, ParquetPageIndexToggle) {
269+
for (const bool enabled : {false, true}) {
270+
for (const bool dictionary : {false, true}) {
271+
for (const auto version :
272+
{TParquetVersion::PARQUET_1_0, TParquetVersion::PARQUET_2_LATEST}) {
273+
TQueryOptions query_options;
274+
query_options.__set_enable_parquet_write_page_index(enabled);
275+
query_options.__set_parquet_write_data_page_size(1024);
276+
RuntimeState state(query_options, TQueryGlobals());
277+
state.set_timezone("UTC");
278+
auto block = page_index_block();
279+
auto exprs = MockSlotRef::create_mock_contexts(block.get_data_types());
280+
io::FileWriterPtr file;
281+
ASSERT_TRUE(_fs->create_file(_file_path, &file).ok());
282+
ParquetFileOptions options {.compression_type = TParquetCompressionType::SNAPPY,
283+
.parquet_version = version,
284+
.parquet_disable_dictionary = !dictionary};
285+
VParquetWriter writer(&state, file.get(), exprs,
286+
std::vector<std::string> {"id", "value", "all_null"}, false,
287+
options);
288+
ASSERT_TRUE(writer.open().ok());
289+
ASSERT_TRUE(writer.write(block).ok());
290+
ASSERT_TRUE(writer.close().ok());
291+
check_page_indexes(enabled);
292+
}
293+
}
294+
}
295+
}
296+
297+
TEST_F(VParquetWriterTest, ParquetPageIndexDataPageVersions) {
298+
for (const auto page_version :
299+
{::parquet::ParquetDataPageVersion::V1, ::parquet::ParquetDataPageVersion::V2}) {
300+
TQueryOptions query_options;
301+
query_options.__set_enable_parquet_write_page_index(true);
302+
query_options.__set_parquet_write_data_page_size(1024);
303+
RuntimeState state(query_options, TQueryGlobals());
304+
auto block = page_index_block();
305+
auto exprs = MockSlotRef::create_mock_contexts(block.get_data_types());
306+
io::FileWriterPtr file;
307+
ASSERT_TRUE(_fs->create_file(_file_path, &file).ok());
308+
const std::vector<std::string> names {"id", "value", "all_null"};
309+
ParquetFileOptions options {.compression_type = TParquetCompressionType::SNAPPY,
310+
.parquet_version = TParquetVersion::PARQUET_2_LATEST};
311+
VParquetWriter writer(&state, file.get(), exprs, names, false, options);
312+
ASSERT_TRUE(writer._parse_properties().ok());
313+
// Exercise both page layouts with the production writer properties; this does not add
314+
// a SQL option for changing the existing data-page version.
315+
auto properties = ::parquet::WriterProperties::Builder(*writer._parquet_writer_properties)
316+
.data_page_version(page_version)
317+
->build();
318+
auto converter = writer._create_arrow_block_convertor(block.get_data_types(), names, "UTC",
319+
cctz::utc_time_zone(), false);
320+
ASSERT_TRUE(converter->init().ok());
321+
std::shared_ptr<arrow::RecordBatch> batch;
322+
ASSERT_TRUE(converter->convert_to_arrow(block, arrow::default_memory_pool(), &batch).ok());
323+
auto result = ::parquet::arrow::FileWriter::Open(
324+
*batch->schema(), arrow::default_memory_pool(), writer._outstream, properties,
325+
writer._arrow_properties);
326+
ASSERT_TRUE(result.ok()) << result.status();
327+
auto arrow_writer = std::move(result).ValueOrDie();
328+
ASSERT_TRUE(arrow_writer->WriteRecordBatch(*batch).ok());
329+
ASSERT_TRUE(arrow_writer->Close().ok());
330+
ASSERT_TRUE(writer._outstream->Close().ok());
331+
check_page_indexes(true);
332+
}
333+
}
334+
335+
TEST_F(VParquetWriterTest, ParquetPageSizePreservesHadoopLz4Limit) {
336+
for (const auto codec : {TParquetCompressionType::LZ4_HADOOP, TParquetCompressionType::LZ4}) {
337+
for (const int32_t requested_size : {65536, 262144}) {
338+
TQueryOptions query_options;
339+
query_options.__set_parquet_write_data_page_size(requested_size);
340+
RuntimeState state(query_options, TQueryGlobals());
341+
state.set_timezone("UTC");
342+
auto block = page_index_block();
343+
auto exprs = MockSlotRef::create_mock_contexts(block.get_data_types());
344+
io::FileWriterPtr file;
345+
ASSERT_TRUE(_fs->create_file(_file_path, &file).ok());
346+
ParquetFileOptions options {.compression_type = codec,
347+
.parquet_version = TParquetVersion::PARQUET_1_0};
348+
VParquetWriter writer(&state, file.get(), exprs,
349+
std::vector<std::string> {"id", "value", "all_null"}, false,
350+
options);
351+
ASSERT_TRUE(writer.open().ok());
352+
EXPECT_TRUE(writer._parquet_writer_properties->page_index_enabled());
353+
EXPECT_EQ(writer._parquet_writer_properties->data_pagesize(),
354+
codec == TParquetCompressionType::LZ4_HADOOP
355+
? std::min(requested_size, 131072)
356+
: requested_size);
357+
ASSERT_TRUE(writer.close().ok());
358+
}
359+
}
360+
}
361+
362+
TEST_F(VParquetWriterTest, HiveAndIcebergParquetPageIndexOptions) {
363+
const std::string json =
364+
R"({"type":"struct","fields":[{"id":1,"name":"id","required":true,"type":"int"},{"id":2,"name":"value","required":false,"type":"int"},{"id":3,"name":"all_null","required":false,"type":"int"}]})";
365+
auto schema = iceberg::SchemaParser::from_json(json);
366+
const std::map<std::string, std::string> hadoop_conf;
367+
for (const bool iceberg_writer : {false, true}) {
368+
for (const bool enabled : {false, true}) {
369+
TQueryOptions query_options;
370+
query_options.__set_enable_parquet_write_page_index(enabled);
371+
query_options.__set_parquet_write_data_page_size(1024);
372+
RuntimeState state(query_options, TQueryGlobals());
373+
state.set_timezone("UTC");
374+
auto block = page_index_block();
375+
auto exprs = MockSlotRef::create_mock_contexts(block.get_data_types());
376+
const auto name = "page_index_" + UniqueId::gen_uid().to_string();
377+
_file_path = "./" + name + "-0.parquet";
378+
TDataSink sink;
379+
if (iceberg_writer) {
380+
VIcebergPartitionWriter writer(sink, {}, exprs, *schema, &json,
381+
{"id", "value", "all_null"},
382+
{.write_path = ".",
383+
.original_write_path = ".",
384+
.target_path = ".",
385+
.file_type = TFileType::FILE_LOCAL,
386+
.broker_addresses = {}},
387+
name, 0, TFileFormatType::FORMAT_PARQUET,
388+
TFileCompressType::PLAIN, hadoop_conf);
389+
ASSERT_TRUE(writer.open(&state, nullptr, nullptr).ok());
390+
ASSERT_TRUE(writer.write(block).ok());
391+
ASSERT_TRUE(writer.close(Status::OK()).ok());
392+
} else {
393+
VHivePartitionWriter writer(sink, "", TUpdateMode::APPEND, exprs,
394+
{"id", "value", "all_null"},
395+
{.write_path = ".",
396+
.original_write_path = ".",
397+
.target_path = ".",
398+
.file_type = TFileType::FILE_LOCAL,
399+
.broker_addresses = {}},
400+
name, 0, TFileFormatType::FORMAT_PARQUET,
401+
TFileCompressType::PLAIN, nullptr, hadoop_conf);
402+
ASSERT_TRUE(writer.open(&state, nullptr).ok());
403+
ASSERT_TRUE(writer.write(block).ok());
404+
ASSERT_TRUE(writer.close(Status::OK()).ok());
405+
}
406+
check_page_indexes(enabled);
407+
ASSERT_TRUE(_fs->delete_file(_file_path).ok());
408+
}
409+
}
410+
}
411+
186412
TEST_F(VParquetWriterTest, IcebergSplitOffsetsIncludeDictionaryPages) {
187413
check_split_offsets(3, true);
188414
}

‎fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -564,6 +564,8 @@ public String toString() {
564564

565565
public static final String ENABLE_PAGE_CACHE = "enable_page_cache";
566566
public static final String ENABLE_PARQUET_FILE_PAGE_CACHE = "enable_parquet_file_page_cache";
567+
public static final String ENABLE_PARQUET_WRITE_PAGE_INDEX = "enable_parquet_write_page_index";
568+
public static final String PARQUET_WRITE_DATA_PAGE_SIZE = "parquet_write_data_page_size";
567569

568570
public static final String DUMP_NEREIDS_MEMO = "dump_nereids_memo";
569571

@@ -2568,6 +2570,19 @@ public Map<String, Action> getForceEagerAggHintMap() {
25682570
needForward = true)
25692571
public boolean enableParquetFilePageCache = true;
25702572

2573+
@VarAttrDef.VarAttr(name = ENABLE_PARQUET_WRITE_PAGE_INDEX, needForward = true,
2574+
description = "Write ColumnIndex and OffsetIndex metadata to new Parquet files. Default is true.")
2575+
public boolean enableParquetWritePageIndex = true;
2576+
2577+
@VarAttrDef.VarAttr(name = PARQUET_WRITE_DATA_PAGE_SIZE, needForward = true,
2578+
checker = "checkParquetWriteDataPageSize",
2579+
description = "Target uncompressed Parquet data-page size in bytes. Must be positive; default is 1048576.")
2580+
public int parquetWriteDataPageSize = 1048576;
2581+
2582+
public void checkParquetWriteDataPageSize(String value) throws Exception {
2583+
checkFieldValue(PARQUET_WRITE_DATA_PAGE_SIZE, 1, value);
2584+
}
2585+
25712586
@VarAttrDef.VarAttr(name = ENABLE_FOLD_NONDETERMINISTIC_FN)
25722587
public boolean enableFoldNondeterministicFn = false;
25732588

@@ -5845,6 +5860,9 @@ public TQueryOptions toThrift() {
58455860
}
58465861

58475862
tResult.setEnableParquetLazyMat(enableParquetLazyMat);
5863+
// Query options carry the same write policy to OUTFILE and catalog sinks.
5864+
tResult.setEnableParquetWritePageIndex(enableParquetWritePageIndex);
5865+
tResult.setParquetWriteDataPageSize(parquetWriteDataPageSize);
58485866
tResult.setEnableOrcLazyMat(enableOrcLazyMat);
58495867
tResult.setEnableParquetFilterByMinMax(enableParquetFilterByMinMax);
58505868
tResult.setEnableParquetFilterByBloomFilter(enableParquetFilterByBloomFilter);

0 commit comments

Comments
 (0)