Skip to content

Commit ac0b0a9

Browse files
committed
apacheGH-51097: Fix Parquet null counts for fixed-width leaves
Signed-off-by: AnuragRaut08 <[email protected]>
1 parent 8c7de93 commit ac0b0a9

2 files changed

Lines changed: 53 additions & 4 deletions

File tree

‎cpp/src/parquet/arrow/arrow_statistics_test.cc‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,51 @@ INSTANTIATE_TEST_SUITE_P(
160160
/*expected_min=*/"z",
161161
/*expected_max=*/"z"}));
162162

163+
TEST(StatisticsTest, FixedWidthLeafUnderListStructNullCount) {
164+
// Null counts for fixed-width leaves under list<struct<...>>
165+
// must include null and empty list entries from the repeated ancestor.
166+
auto schema = ::arrow::schema({::arrow::field(
167+
"col", ::arrow::list(::arrow::struct_(
168+
{::arrow::field("s", ::arrow::utf8()),
169+
::arrow::field("i32", ::arrow::int32())})))});
170+
171+
auto table = ::arrow::Table::Make(
172+
schema,
173+
{::arrow::ArrayFromJSON(
174+
::arrow::list(::arrow::struct_(
175+
{::arrow::field("s", ::arrow::utf8()),
176+
::arrow::field("i32", ::arrow::int32())})),
177+
R"([[{"s":"a","i32":1}],null,[],[{"s":null,"i32":null},{"s":"b","i32":2}]])")});
178+
179+
std::shared_ptr<::arrow::ResizableBuffer> serialized_data = AllocateBuffer();
180+
auto out_stream =
181+
std::make_shared<::arrow::io::BufferOutputStream>(serialized_data);
182+
183+
ASSERT_OK_AND_ASSIGN(
184+
std::unique_ptr<FileWriter> writer,
185+
FileWriter::Open(*schema, default_memory_pool(), out_stream,
186+
default_writer_properties(),
187+
default_arrow_writer_properties()));
188+
ASSERT_OK(writer->WriteTable(*table));
189+
ASSERT_OK(writer->Close());
190+
ASSERT_OK(out_stream->Close());
191+
192+
auto buffer_reader = std::make_shared<::arrow::io::BufferReader>(serialized_data);
193+
auto parquet_reader = ParquetFileReader::Open(std::move(buffer_reader));
194+
auto metadata = parquet_reader->metadata();
195+
auto row_group = metadata->RowGroup(0);
196+
197+
ASSERT_EQ(row_group->num_columns(), 2);
198+
199+
auto int32_stats = row_group->ColumnChunk(1)->statistics();
200+
ASSERT_NE(int32_stats, nullptr);
201+
202+
// Fixed-width leaves must include nulls from repeated ancestors
203+
// (e.g. null or empty lists) in the column statistics.
204+
EXPECT_EQ(int32_stats->null_count(), 3);
205+
EXPECT_EQ(int32_stats->num_values(), 2);
206+
}
207+
163208
TEST(StatisticsTest, TruncateOnlyHalfMinMax) {
164209
// GH-43382: Tests when we only have min or max, the `HasMinMax` should be false.
165210
std::shared_ptr<::arrow::ResizableBuffer> serialized_data = AllocateBuffer();

‎cpp/src/parquet/column_writer.cc‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1394,6 +1394,7 @@ class TypedColumnWriterImpl : public ColumnWriterImpl,
13941394
MaybeCalculateValidityBits(AddIfNotNull(def_levels, offset), batch_size,
13951395
&batch_num_values, &batch_num_spaced_values,
13961396
&null_count);
1397+
const int64_t total_null_count = batch_size - batch_num_values;
13971398

13981399
WriteLevelsSpaced(batch_size, AddIfNotNull(def_levels, offset),
13991400
AddIfNotNull(rep_levels, offset));
@@ -1407,8 +1408,8 @@ class TypedColumnWriterImpl : public ColumnWriterImpl,
14071408
valid_bits_offset + value_offset, /*num_levels=*/batch_size,
14081409
null_count);
14091410
}
1410-
CommitWriteAndCheckPageLimit(batch_size, batch_num_spaced_values, null_count,
1411-
check_page);
1411+
CommitWriteAndCheckPageLimit(batch_size, batch_num_spaced_values,
1412+
total_null_count, check_page);
14121413
value_offset += batch_num_spaced_values;
14131414

14141415
// Dictionary size checked separately from data page size since we
@@ -1750,7 +1751,8 @@ class TypedColumnWriterImpl : public ColumnWriterImpl,
17501751
internal::DefLevelsToBitmap(def_levels, batch_size, level_info_, &io);
17511752
*out_values_to_write = io.values_read - io.null_count;
17521753
*out_spaced_values_to_write = io.values_read;
1753-
*null_count = io.null_count;
1754+
// io.null_count excludes nulls from repeated ancestors.
1755+
*null_count = batch_size - *out_values_to_write;
17541756
}
17551757

17561758
Result<std::shared_ptr<Array>> MaybeReplaceValidity(std::shared_ptr<Array> array,
@@ -2038,6 +2040,7 @@ Status TypedColumnWriterImpl<ParquetType>::WriteArrowDictionary(
20382040
// had so we need to recompute it from def levels.
20392041
MaybeCalculateValidityBits(AddIfNotNull(def_levels, offset), batch_size,
20402042
&batch_num_values, &batch_num_spaced_values, &null_count);
2043+
const int64_t total_null_count = batch_size - batch_num_values;
20412044
WriteLevelsSpaced(batch_size, AddIfNotNull(def_levels, offset),
20422045
AddIfNotNull(rep_levels, offset));
20432046
std::shared_ptr<Array> writeable_indices =
@@ -2051,7 +2054,8 @@ Status TypedColumnWriterImpl<ParquetType>::WriteArrowDictionary(
20512054
dict_encoder->PutIndices(*writeable_indices);
20522055
// Update unencoded byte array data size to size statistics
20532056
UpdateUnencodedDataBytes();
2054-
CommitWriteAndCheckPageLimit(batch_size, batch_num_values, null_count, check_page);
2057+
CommitWriteAndCheckPageLimit(batch_size, batch_num_values, total_null_count,
2058+
check_page);
20552059
value_offset += batch_num_spaced_values;
20562060
};
20572061

0 commit comments

Comments
 (0)