Skip to content

Commit 51955ec

Browse files
AnuragRaut08wgtmac
andauthored
GH-51097: Fix Parquet null counts for fixed-width leaves (#51357)
### Rationale for this change Parquet null-count statistics can be incorrect for fixed-width leaf columns nested below a repeated ancestor, such as `list<struct<...>>`. When a list is null or empty, its descendant leaf does not produce a value in the leaf values buffer. `DefLevelsToBitmap` intentionally excludes these repeated-ancestor nulls from its `null_count`. `MaybeCalculateValidityBits` was using that value directly for the Parquet column statistics, causing the null count to be undercounted. For example, a `list<struct<string, int32>>` column can report an incorrect `null_count` for the `int32` leaf even though the encoded definition levels correctly represent the null and empty list entries. ### What changes are included in this PR? * Update `MaybeCalculateValidityBits` to calculate the total null count from `batch_size - out_values_to_write`. * This includes nulls represented by repeated ancestors, such as null or empty lists, while preserving the existing validity bitmap behavior. * Add a regression test covering a fixed-width `int32` leaf under `list<struct<...>>`. ### Are these changes tested? Yes. Added a regression test that verifies the `int32` leaf reports: * `null_count = 3` * `num_values = 2` The focused regression test passes: `StatisticsTest.FixedWidthLeafUnderListStructNullCount` The existing `parquet-writer-test` also passes. ### Are there any user-facing changes? Yes. This fixes incorrect Parquet column statistics for affected nested fixed-width columns. The change does not alter the encoded data or public APIs. ### This PR contains a "Critical Fix". This fixes a bug that produces incorrect Parquet statistics. The underlying data remains correct, but the reported `null_count` for affected fixed-width leaf columns can be incorrect. ### Was AI used for this PR? **PR code and description written by:** * [x] Human * [ ] AI **Reviewed before submission by:** * [x] Human * [ ] AI * [ ] Not reviewed * GitHub Issue: #51097 Lead-authored-by: AnuragRaut08 <[email protected]> Co-authored-by: Gang Wu <[email protected]> Signed-off-by: Gang Wu <[email protected]>
1 parent 3ad410b commit 51955ec

2 files changed

Lines changed: 111 additions & 5 deletions

File tree

‎cpp/src/parquet/arrow/arrow_statistics_test.cc‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include "arrow/array.h"
2121
#include "arrow/array/builder_primitive.h"
2222
#include "arrow/array/builder_time.h"
23+
#include "arrow/compute/api.h"
2324
#include "arrow/table.h"
2425
#include "arrow/testing/gtest_util.h"
2526

@@ -160,6 +161,108 @@ INSTANTIATE_TEST_SUITE_P(
160161
/*expected_min=*/"z",
161162
/*expected_max=*/"z"}));
162163

164+
TEST(StatisticsTest, FixedWidthLeafUnderListStructNullCount) {
165+
// Null counts for leaves under list<struct<...>> must include null and empty
166+
// list entries from the repeated ancestor.
167+
for (const auto data_page_version :
168+
{ParquetDataPageVersion::V1, ParquetDataPageVersion::V2}) {
169+
for (const bool use_dictionary : {false, true}) {
170+
SCOPED_TRACE(::testing::Message()
171+
<< "data_page_version=" << static_cast<int>(data_page_version)
172+
<< ", use_dictionary=" << use_dictionary);
173+
174+
auto string_type = use_dictionary
175+
? ::arrow::dictionary(::arrow::int32(), ::arrow::utf8())
176+
: ::arrow::utf8();
177+
auto list_type = ::arrow::list(::arrow::struct_(
178+
{::arrow::field("s", string_type), ::arrow::field("i32", ::arrow::int32())}));
179+
auto schema = ::arrow::schema({::arrow::field("col", list_type)});
180+
auto table = ::arrow::TableFromJSON(schema, {R"([
181+
[[{"s":"a","i32":1}]],
182+
[null],
183+
[[]],
184+
[[{"s":null,"i32":null},{"s":"b","i32":2}]]
185+
])"});
186+
187+
WriterProperties::Builder properties_builder;
188+
properties_builder.data_page_version(data_page_version);
189+
if (use_dictionary) {
190+
properties_builder.enable_dictionary();
191+
} else {
192+
properties_builder.disable_dictionary();
193+
}
194+
195+
std::shared_ptr<::arrow::ResizableBuffer> serialized_data = AllocateBuffer();
196+
auto out_stream =
197+
std::make_shared<::arrow::io::BufferOutputStream>(serialized_data);
198+
199+
ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileWriter> writer,
200+
FileWriter::Open(*schema, default_memory_pool(), out_stream,
201+
properties_builder.build(),
202+
default_arrow_writer_properties()));
203+
ASSERT_OK(writer->WriteTable(*table));
204+
ASSERT_OK(writer->Close());
205+
ASSERT_OK(out_stream->Close());
206+
207+
auto buffer_reader = std::make_shared<::arrow::io::BufferReader>(serialized_data);
208+
auto parquet_reader = ParquetFileReader::Open(std::move(buffer_reader));
209+
auto metadata = parquet_reader->metadata();
210+
auto row_group = metadata->RowGroup(0);
211+
212+
ASSERT_EQ(row_group->num_columns(), 2);
213+
214+
for (int i = 0; i < 2; ++i) {
215+
auto stats = row_group->ColumnChunk(i)->statistics();
216+
ASSERT_NE(stats, nullptr);
217+
EXPECT_EQ(stats->null_count(), 3);
218+
EXPECT_EQ(stats->num_values(), 2);
219+
220+
if (data_page_version == ParquetDataPageVersion::V2) {
221+
auto page_reader = parquet_reader->RowGroup(0)->GetColumnPageReader(i);
222+
if (use_dictionary) {
223+
auto dictionary_page = page_reader->NextPage();
224+
ASSERT_NE(dictionary_page, nullptr);
225+
ASSERT_EQ(dictionary_page->type(), PageType::DICTIONARY_PAGE);
226+
}
227+
auto page = page_reader->NextPage();
228+
ASSERT_NE(page, nullptr);
229+
ASSERT_EQ(page->type(), PageType::DATA_PAGE_V2);
230+
auto data_page = std::static_pointer_cast<DataPageV2>(page);
231+
EXPECT_EQ(data_page->num_values(), 5);
232+
EXPECT_EQ(data_page->num_nulls(), 3);
233+
EXPECT_EQ(page_reader->NextPage(), nullptr);
234+
}
235+
}
236+
237+
ASSERT_OK_AND_ASSIGN(
238+
auto file_reader,
239+
FileReader::Make(default_memory_pool(), std::move(parquet_reader),
240+
default_arrow_reader_properties()));
241+
242+
ASSERT_OK_AND_ASSIGN(auto read_table, file_reader->ReadTable());
243+
244+
if (use_dictionary) {
245+
auto plain_list_type =
246+
::arrow::list(::arrow::struct_({::arrow::field("s", ::arrow::utf8()),
247+
::arrow::field("i32", ::arrow::int32())}));
248+
249+
ASSERT_OK_AND_ASSIGN(
250+
auto read_array,
251+
::arrow::compute::Cast(read_table->column(0)->chunk(0), plain_list_type));
252+
253+
auto expected_array = table->column(0)->chunk(0);
254+
255+
ASSERT_OK_AND_ASSIGN(auto expected_plain,
256+
::arrow::compute::Cast(expected_array, plain_list_type));
257+
258+
ASSERT_TRUE(read_array.Equals(expected_plain));
259+
} else {
260+
ASSERT_TRUE(read_table->Equals(*table));
261+
}
262+
}
263+
}
264+
}
265+
163266
TEST(StatisticsTest, TruncateOnlyHalfMinMax) {
164267
// GH-43382: Tests when we only have min or max, the `HasMinMax` should be false.
165268
std::shared_ptr<::arrow::ResizableBuffer> serialized_data = AllocateBuffer();

‎cpp/src/parquet/column_writer.cc‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1397,18 +1397,19 @@ class TypedColumnWriterImpl : public ColumnWriterImpl,
13971397

13981398
WriteLevelsSpaced(batch_size, AddIfNotNull(def_levels, offset),
13991399
AddIfNotNull(rep_levels, offset));
1400+
const int64_t parquet_null_count = batch_size - batch_num_values;
14001401
if (bits_buffer_ != nullptr) {
14011402
WriteValuesSpaced(AddIfNotNull(values, value_offset), batch_num_values,
14021403
batch_num_spaced_values, bits_buffer_->data(), /*offset=*/0,
1403-
/*num_levels=*/batch_size, null_count);
1404+
/*num_levels=*/batch_size, parquet_null_count);
14041405
} else {
14051406
WriteValuesSpaced(AddIfNotNull(values, value_offset), batch_num_values,
14061407
batch_num_spaced_values, valid_bits,
14071408
valid_bits_offset + value_offset, /*num_levels=*/batch_size,
1408-
null_count);
1409+
parquet_null_count);
14091410
}
1410-
CommitWriteAndCheckPageLimit(batch_size, batch_num_spaced_values, null_count,
1411-
check_page);
1411+
CommitWriteAndCheckPageLimit(batch_size, batch_num_spaced_values,
1412+
parquet_null_count, check_page);
14121413
value_offset += batch_num_spaced_values;
14131414

14141415
// Dictionary size checked separately from data page size since we
@@ -2051,7 +2052,9 @@ Status TypedColumnWriterImpl<ParquetType>::WriteArrowDictionary(
20512052
dict_encoder->PutIndices(*writeable_indices);
20522053
// Update unencoded byte array data size to size statistics
20532054
UpdateUnencodedDataBytes();
2054-
CommitWriteAndCheckPageLimit(batch_size, batch_num_values, null_count, check_page);
2055+
const int64_t parquet_null_count = batch_size - batch_num_values;
2056+
CommitWriteAndCheckPageLimit(batch_size, batch_num_values, parquet_null_count,
2057+
check_page);
20552058
value_offset += batch_num_spaced_values;
20562059
};
20572060

0 commit comments

Comments
 (0)