Skip to content

Commit d6f477d

Browse files
committed
apacheGH-51097: Fix Parquet null counts for fixed-width leaves
Signed-off-by: AnuragRaut08 <[email protected]>
1 parent 8c7de93 commit d6f477d

2 files changed

Lines changed: 46 additions & 1 deletion

File tree

‎cpp/src/parquet/arrow/arrow_statistics_test.cc‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,50 @@ INSTANTIATE_TEST_SUITE_P(
160160
/*expected_min=*/"z",
161161
/*expected_max=*/"z"}));
162162

163+
TEST(StatisticsTest, FixedWidthLeafUnderListStructNullCount) {
164+
// Null counts for fixed-width leaves under list<struct<...>>
165+
// must include null and empty list entries from the repeated ancestor.
166+
auto schema = ::arrow::schema({::arrow::field(
167+
"col", ::arrow::list(::arrow::struct_(
168+
{::arrow::field("s", ::arrow::utf8()),
169+
::arrow::field("i32", ::arrow::int32())})))});
170+
171+
auto table = Table::Make(
172+
schema,
173+
{ArrayFromJSON(
174+
schema->field(0)->type(),
175+
R"([[{"s":"a","i32":1}],null,[],[{"s":null,"i32":null},{"s":"b","i32":2}]])")});
176+
177+
std::shared_ptr<::arrow::ResizableBuffer> serialized_data = AllocateBuffer();
178+
auto out_stream =
179+
std::make_shared<::arrow::io::BufferOutputStream>(serialized_data);
180+
181+
ASSERT_OK_AND_ASSIGN(
182+
std::unique_ptr<FileWriter> writer,
183+
FileWriter::Open(*schema, default_memory_pool(), out_stream,
184+
default_writer_properties(),
185+
default_arrow_writer_properties()));
186+
ASSERT_OK(writer->WriteTable(*table, std::numeric_limits<int64_t>::max()));
187+
ASSERT_OK(writer->Close());
188+
ASSERT_OK(out_stream->Close());
189+
190+
auto buffer_reader = std::make_shared<::arrow::io::BufferReader>(serialized_data);
191+
auto parquet_reader = ParquetFileReader::Open(std::move(buffer_reader));
192+
auto metadata = parquet_reader->metadata();
193+
auto row_group = metadata->RowGroup(0);
194+
195+
ASSERT_EQ(row_group->num_columns(), 2);
196+
197+
auto int32_stats = row_group->ColumnChunk(1)->statistics();
198+
199+
ASSERT_NE(int32_stats, nullptr);
200+
201+
// Fixed-width leaves must include nulls from repeated ancestors
202+
// (e.g. null or empty lists) in the column statistics.
203+
EXPECT_EQ(int32_stats->null_count(), 3);
204+
EXPECT_EQ(int32_stats->num_values(), 2);
205+
}
206+
163207
TEST(StatisticsTest, TruncateOnlyHalfMinMax) {
164208
// GH-43382: Tests when we only have min or max, the `HasMinMax` should be false.
165209
std::shared_ptr<::arrow::ResizableBuffer> serialized_data = AllocateBuffer();

‎cpp/src/parquet/column_writer.cc‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1750,7 +1750,8 @@ class TypedColumnWriterImpl : public ColumnWriterImpl,
17501750
internal::DefLevelsToBitmap(def_levels, batch_size, level_info_, &io);
17511751
*out_values_to_write = io.values_read - io.null_count;
17521752
*out_spaced_values_to_write = io.values_read;
1753-
*null_count = io.null_count;
1753+
// Include nulls from repeated ancestors, which are excluded from io.null_count.
1754+
*null_count = batch_size - *out_values_to_write;
17541755
}
17551756

17561757
Result<std::shared_ptr<Array>> MaybeReplaceValidity(std::shared_ptr<Array> array,

0 commit comments

Comments
 (0)