Skip to content

Commit fee8bf9

Browse files
mapleFUpitrouwgtmac
authored
GH-35757: [C++][Parquet] using page-encoding-stats to build encodings (#35758)
### Rationale for this change This patch uses `encoding_stats` to build the encodings. Currently, `ColumnChunk.encodings` is repeated ( May has multiple `PLAIN` ) and having some wired logic by getting `WriterProperties`. And if later we want to support Fallback to non "PLAIN" encoding, `encoding_stats` would be right, but `encoding` would be "PLAIN". So this patch using `encoding_stats` to build the `encoding`, and force them to be consistent. ### What changes are included in this PR? ### Are these changes tested? Already has test ### Are there any user-facing changes? Some behavior would be changed, but I think my change is right :) * Closes: #35757 Lead-authored-by: mwish <[email protected]> Co-authored-by: Antoine Pitrou <[email protected]> Co-authored-by: Gang Wu <[email protected]> Signed-off-by: Antoine Pitrou <[email protected]>
1 parent 5c772af commit fee8bf9

4 files changed

Lines changed: 69 additions & 39 deletions

File tree

‎cpp/src/parquet/column_writer_test.cc‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -184,24 +184,25 @@ class TestPrimitiveWriter : public PrimitiveTypedTest<TestType> {
184184
ASSERT_EQ(VERY_LARGE_SIZE, this->values_read_);
185185
this->values_.resize(VERY_LARGE_SIZE);
186186
ASSERT_EQ(this->values_, this->values_out_);
187-
std::vector<Encoding::type> encodings = this->metadata_encodings();
187+
std::vector<Encoding::type> encodings_vector = this->metadata_encodings();
188+
std::set<Encoding::type> encodings(encodings_vector.begin(), encodings_vector.end());
188189

189190
if (this->type_num() == Type::BOOLEAN) {
190191
// Dictionary encoding is not allowed for boolean type
191192
// There are 2 encodings (PLAIN, RLE) in a non dictionary encoding case
192-
std::vector<Encoding::type> expected({Encoding::PLAIN, Encoding::RLE});
193+
std::set<Encoding::type> expected({Encoding::PLAIN, Encoding::RLE});
193194
ASSERT_EQ(encodings, expected);
194195
} else if (version == ParquetVersion::PARQUET_1_0) {
195-
// There are 4 encodings (PLAIN_DICTIONARY, PLAIN, RLE, PLAIN) in a fallback case
196+
// There are 3 encodings (PLAIN_DICTIONARY, PLAIN, RLE) in a fallback case
196197
// for version 1.0
197-
std::vector<Encoding::type> expected(
198-
{Encoding::PLAIN_DICTIONARY, Encoding::PLAIN, Encoding::RLE, Encoding::PLAIN});
198+
std::set<Encoding::type> expected(
199+
{Encoding::PLAIN_DICTIONARY, Encoding::PLAIN, Encoding::RLE});
199200
ASSERT_EQ(encodings, expected);
200201
} else {
201-
// There are 4 encodings (RLE_DICTIONARY, PLAIN, RLE, PLAIN) in a fallback case for
202+
// There are 3 encodings (RLE_DICTIONARY, PLAIN, RLE) in a fallback case for
202203
// version 2.0
203-
std::vector<Encoding::type> expected(
204-
{Encoding::RLE_DICTIONARY, Encoding::PLAIN, Encoding::RLE, Encoding::PLAIN});
204+
std::set<Encoding::type> expected(
205+
{Encoding::RLE_DICTIONARY, Encoding::PLAIN, Encoding::RLE});
205206
ASSERT_EQ(encodings, expected);
206207
}
207208

‎cpp/src/parquet/metadata.cc‎

Lines changed: 28 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1462,40 +1462,44 @@ class ColumnChunkMetaDataBuilder::ColumnChunkMetaDataBuilderImpl {
14621462
column_chunk_->meta_data.__set_total_compressed_size(compressed_size);
14631463

14641464
std::vector<format::Encoding::type> thrift_encodings;
1465-
if (has_dictionary) {
1466-
thrift_encodings.push_back(ToThrift(properties_->dictionary_index_encoding()));
1467-
if (properties_->version() == ParquetVersion::PARQUET_1_0) {
1468-
thrift_encodings.push_back(ToThrift(Encoding::PLAIN));
1469-
} else {
1470-
thrift_encodings.push_back(ToThrift(properties_->dictionary_page_encoding()));
1471-
}
1472-
} else { // Dictionary not enabled
1473-
thrift_encodings.push_back(ToThrift(properties_->encoding(column_->path())));
1474-
}
1475-
thrift_encodings.push_back(ToThrift(Encoding::RLE));
1476-
// Only PLAIN encoding is supported for fallback in V1
1477-
// TODO(majetideepak): Use user specified encoding for V2
1478-
if (dictionary_fallback) {
1479-
thrift_encodings.push_back(ToThrift(Encoding::PLAIN));
1480-
}
1481-
column_chunk_->meta_data.__set_encodings(thrift_encodings);
14821465
std::vector<format::PageEncodingStats> thrift_encoding_stats;
1466+
auto add_encoding = [&thrift_encodings](format::Encoding::type value) {
1467+
auto it = std::find(thrift_encodings.begin(), thrift_encodings.end(), value);
1468+
if (it == thrift_encodings.end()) {
1469+
thrift_encodings.push_back(value);
1470+
}
1471+
};
14831472
// Add dictionary page encoding stats
1484-
for (const auto& entry : dict_encoding_stats) {
1485-
format::PageEncodingStats dict_enc_stat;
1486-
dict_enc_stat.__set_page_type(format::PageType::DICTIONARY_PAGE);
1487-
dict_enc_stat.__set_encoding(ToThrift(entry.first));
1488-
dict_enc_stat.__set_count(entry.second);
1489-
thrift_encoding_stats.push_back(dict_enc_stat);
1473+
if (has_dictionary) {
1474+
for (const auto& entry : dict_encoding_stats) {
1475+
format::PageEncodingStats dict_enc_stat;
1476+
dict_enc_stat.__set_page_type(format::PageType::DICTIONARY_PAGE);
1477+
// Dictionary Encoding would be PLAIN_DICTIONARY in v1 and
1478+
// PLAIN in v2.
1479+
format::Encoding::type dict_encoding = ToThrift(entry.first);
1480+
dict_enc_stat.__set_encoding(dict_encoding);
1481+
dict_enc_stat.__set_count(entry.second);
1482+
thrift_encoding_stats.push_back(dict_enc_stat);
1483+
add_encoding(dict_encoding);
1484+
}
14901485
}
1486+
// Always add encoding for RL/DL.
1487+
// BIT_PACKED is supported in `LevelEncoder`, but would only be used
1488+
// in benchmark and testing.
1489+
// And for now, we always add RLE even if there are no levels at all,
1490+
// while parquet-mr is more fine-grained.
1491+
add_encoding(format::Encoding::RLE);
14911492
// Add data page encoding stats
14921493
for (const auto& entry : data_encoding_stats) {
14931494
format::PageEncodingStats data_enc_stat;
14941495
data_enc_stat.__set_page_type(format::PageType::DATA_PAGE);
1495-
data_enc_stat.__set_encoding(ToThrift(entry.first));
1496+
format::Encoding::type data_encoding = ToThrift(entry.first);
1497+
data_enc_stat.__set_encoding(data_encoding);
14961498
data_enc_stat.__set_count(entry.second);
14971499
thrift_encoding_stats.push_back(data_enc_stat);
1500+
add_encoding(data_encoding);
14981501
}
1502+
column_chunk_->meta_data.__set_encodings(thrift_encodings);
14991503
column_chunk_->meta_data.__set_encoding_stats(thrift_encoding_stats);
15001504

15011505
const auto& encrypt_md =

‎cpp/src/parquet/metadata_test.cc‎

Lines changed: 31 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,6 @@ std::unique_ptr<parquet::FileMetaData> GenerateTableMetaData(
6666
// column metadata
6767
col1_builder->SetStatistics(stats_int);
6868
col2_builder->SetStatistics(stats_float);
69-
dict_encoding_stats.clear();
7069
col1_builder->Finish(nrows / 2, /*dictionary_page_offset=*/0, 0, 10, 512, 600,
7170
/*has_dictionary=*/false, false, dict_encoding_stats,
7271
data_encoding_stats);
@@ -80,6 +79,13 @@ std::unique_ptr<parquet::FileMetaData> GenerateTableMetaData(
8079
return f_builder->Finish();
8180
}
8281

82+
void AssertEncodings(const ColumnChunkMetaData& data,
83+
const std::set<parquet::Encoding::type>& expected) {
84+
std::set<parquet::Encoding::type> encodings(data.encodings().begin(),
85+
data.encodings().end());
86+
ASSERT_EQ(encodings, expected);
87+
}
88+
8389
TEST(Metadata, TestBuildAccess) {
8490
parquet::schema::NodeVector fields;
8591
parquet::schema::NodePtr root;
@@ -160,8 +166,18 @@ TEST(Metadata, TestBuildAccess) {
160166
ASSERT_EQ(DEFAULT_COMPRESSION_TYPE, rg1_column2->compression());
161167
ASSERT_EQ(nrows / 2, rg1_column1->num_values());
162168
ASSERT_EQ(nrows / 2, rg1_column2->num_values());
163-
ASSERT_EQ(3, rg1_column1->encodings().size());
164-
ASSERT_EQ(3, rg1_column2->encodings().size());
169+
{
170+
std::set<parquet::Encoding::type> encodings{parquet::Encoding::RLE,
171+
parquet::Encoding::RLE_DICTIONARY,
172+
parquet::Encoding::PLAIN};
173+
AssertEncodings(*rg1_column1, encodings);
174+
}
175+
{
176+
std::set<parquet::Encoding::type> encodings{parquet::Encoding::RLE,
177+
parquet::Encoding::RLE_DICTIONARY,
178+
parquet::Encoding::PLAIN};
179+
AssertEncodings(*rg1_column2, encodings);
180+
}
165181
ASSERT_EQ(512, rg1_column1->total_compressed_size());
166182
ASSERT_EQ(512, rg1_column2->total_compressed_size());
167183
ASSERT_EQ(600, rg1_column1->total_uncompressed_size());
@@ -197,8 +213,17 @@ TEST(Metadata, TestBuildAccess) {
197213
ASSERT_EQ(nrows / 2, rg2_column2->num_values());
198214
ASSERT_EQ(DEFAULT_COMPRESSION_TYPE, rg2_column1->compression());
199215
ASSERT_EQ(DEFAULT_COMPRESSION_TYPE, rg2_column2->compression());
200-
ASSERT_EQ(2, rg2_column1->encodings().size());
201-
ASSERT_EQ(3, rg2_column2->encodings().size());
216+
{
217+
std::set<parquet::Encoding::type> encodings{parquet::Encoding::RLE,
218+
parquet::Encoding::PLAIN};
219+
AssertEncodings(*rg2_column1, encodings);
220+
}
221+
{
222+
std::set<parquet::Encoding::type> encodings{parquet::Encoding::RLE,
223+
parquet::Encoding::RLE_DICTIONARY,
224+
parquet::Encoding::PLAIN};
225+
AssertEncodings(*rg2_column2, encodings);
226+
}
202227
ASSERT_EQ(512, rg2_column1->total_compressed_size());
203228
ASSERT_EQ(512, rg2_column2->total_compressed_size());
204229
ASSERT_EQ(600, rg2_column1->total_uncompressed_size());
@@ -209,7 +234,7 @@ TEST(Metadata, TestBuildAccess) {
209234
ASSERT_EQ(10, rg2_column1->data_page_offset());
210235
ASSERT_EQ(26, rg2_column2->data_page_offset());
211236
ASSERT_EQ(2, rg2_column1->encoding_stats().size());
212-
ASSERT_EQ(2, rg2_column2->encoding_stats().size());
237+
ASSERT_EQ(3, rg2_column2->encoding_stats().size());
213238

214239
// Test FileMetaData::set_file_path
215240
ASSERT_TRUE(rg2_column1->file_path().empty());

‎python/pyarrow/tests/parquet/test_metadata.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ def test_parquet_metadata_api():
128128
assert col_meta.is_stats_set is True
129129
assert isinstance(col_meta.statistics, pq.Statistics)
130130
assert col_meta.compression == 'SNAPPY'
131-
assert col_meta.encodings == ('PLAIN', 'RLE')
131+
assert set(col_meta.encodings) == {'PLAIN', 'RLE'}
132132
assert col_meta.has_dictionary_page is False
133133
assert col_meta.dictionary_page_offset is None
134134
assert col_meta.data_page_offset > 0

0 commit comments

Comments
 (0)