Skip to content

Commit ea6e4b1

Browse files
committed
apacheGH-51097: Fix Parquet null counts for fixed-width leaves
1 parent 49995ad commit ea6e4b1

2 files changed

Lines changed: 91 additions & 48 deletions

File tree

‎cpp/src/parquet/arrow/arrow_statistics_test.cc‎

Lines changed: 85 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include "gtest/gtest.h"
1919

2020
#include "arrow/array.h"
21+
#include "arrow/compute/api.h"
2122
#include "arrow/array/builder_primitive.h"
2223
#include "arrow/array/builder_time.h"
2324
#include "arrow/table.h"
@@ -161,48 +162,90 @@ INSTANTIATE_TEST_SUITE_P(
161162
/*expected_max=*/"z"}));
162163

163164
TEST(StatisticsTest, FixedWidthLeafUnderListStructNullCount) {
164-
// Null counts for fixed-width leaves under list<struct<...>>
165-
// must include null and empty list entries from the repeated ancestor.
166-
auto schema = ::arrow::schema({::arrow::field(
167-
"col", ::arrow::list(::arrow::struct_(
168-
{::arrow::field("s", ::arrow::utf8()),
169-
::arrow::field("i32", ::arrow::int32())})))});
170-
171-
auto table = ::arrow::Table::Make(
172-
schema,
173-
{::arrow::ArrayFromJSON(
174-
::arrow::list(::arrow::struct_(
175-
{::arrow::field("s", ::arrow::utf8()),
176-
::arrow::field("i32", ::arrow::int32())})),
177-
R"([[{"s":"a","i32":1}],null,[],[{"s":null,"i32":null},{"s":"b","i32":2}]])")});
178-
179-
std::shared_ptr<::arrow::ResizableBuffer> serialized_data = AllocateBuffer();
180-
auto out_stream =
181-
std::make_shared<::arrow::io::BufferOutputStream>(serialized_data);
182-
183-
ASSERT_OK_AND_ASSIGN(
184-
std::unique_ptr<FileWriter> writer,
185-
FileWriter::Open(*schema, default_memory_pool(), out_stream,
186-
default_writer_properties(),
187-
default_arrow_writer_properties()));
188-
ASSERT_OK(writer->WriteTable(*table));
189-
ASSERT_OK(writer->Close());
190-
ASSERT_OK(out_stream->Close());
191-
192-
auto buffer_reader = std::make_shared<::arrow::io::BufferReader>(serialized_data);
193-
auto parquet_reader = ParquetFileReader::Open(std::move(buffer_reader));
194-
auto metadata = parquet_reader->metadata();
195-
auto row_group = metadata->RowGroup(0);
196-
197-
ASSERT_EQ(row_group->num_columns(), 2);
198-
199-
auto int32_stats = row_group->ColumnChunk(1)->statistics();
200-
ASSERT_NE(int32_stats, nullptr);
201-
202-
// Fixed-width leaves must include nulls from repeated ancestors
203-
// (e.g. null or empty lists) in the column statistics.
204-
EXPECT_EQ(int32_stats->null_count(), 3);
205-
EXPECT_EQ(int32_stats->num_values(), 2);
165+
// Null counts for leaves under list<struct<...>> must include null and empty
166+
// list entries from the repeated ancestor.
167+
for (const auto data_page_version :
168+
{ParquetDataPageVersion::V1, ParquetDataPageVersion::V2}) {
169+
for (const bool use_dictionary : {false, true}) {
170+
SCOPED_TRACE(::testing::Message()
171+
<< "data_page_version=" << static_cast<int>(data_page_version)
172+
<< ", use_dictionary=" << use_dictionary);
173+
174+
auto string_type =
175+
use_dictionary ? ::arrow::dictionary(::arrow::int32(), ::arrow::utf8())
176+
: ::arrow::utf8();
177+
auto list_type = ::arrow::list(::arrow::struct_(
178+
{::arrow::field("s", string_type),
179+
::arrow::field("i32", ::arrow::int32())}));
180+
auto schema = ::arrow::schema({::arrow::field("col", list_type)});
181+
182+
auto table = Table::Make(
183+
schema, {ArrayFromJSON(
184+
list_type,
185+
R"([[{"s":"a","i32":1}],null,[],[{"s":null,"i32":null},{"s":"b","i32":2}]])")});
186+
187+
WriterProperties::Builder properties_builder;
188+
properties_builder.data_page_version(data_page_version);
189+
if (use_dictionary) {
190+
properties_builder.enable_dictionary();
191+
}
192+
193+
std::shared_ptr<::arrow::ResizableBuffer> serialized_data = AllocateBuffer();
194+
auto out_stream =
195+
std::make_shared<::arrow::io::BufferOutputStream>(serialized_data);
196+
197+
ASSERT_OK_AND_ASSIGN(
198+
std::unique_ptr<FileWriter> writer,
199+
FileWriter::Open(*schema, default_memory_pool(), out_stream,
200+
properties_builder.build(),
201+
default_arrow_writer_properties()));
202+
ASSERT_OK(writer->WriteTable(*table));
203+
ASSERT_OK(writer->Close());
204+
ASSERT_OK(out_stream->Close());
205+
206+
auto buffer_reader = std::make_shared<::arrow::io::BufferReader>(serialized_data);
207+
auto parquet_reader = ParquetFileReader::Open(std::move(buffer_reader));
208+
auto metadata = parquet_reader->metadata();
209+
auto row_group = metadata->RowGroup(0);
210+
211+
ASSERT_EQ(row_group->num_columns(), 2);
212+
213+
for (int i = 0; i < 2; ++i) {
214+
auto stats = row_group->ColumnChunk(i)->statistics();
215+
ASSERT_NE(stats, nullptr);
216+
EXPECT_EQ(stats->null_count(), 3);
217+
EXPECT_EQ(stats->num_values(), 2);
218+
}
219+
220+
ASSERT_OK_AND_ASSIGN(
221+
auto file_reader,
222+
FileReader::Make(default_memory_pool(), std::move(parquet_reader),
223+
default_arrow_reader_properties()));
224+
225+
ASSERT_OK_AND_ASSIGN(auto read_table, file_reader->ReadTable());
226+
227+
if (use_dictionary) {
228+
auto plain_list_type = ::arrow::list(::arrow::struct_(
229+
{::arrow::field("s", ::arrow::utf8()),
230+
::arrow::field("i32", ::arrow::int32())}));
231+
232+
ASSERT_OK_AND_ASSIGN(
233+
auto read_array,
234+
::arrow::compute::Cast(read_table->column(0)->chunk(0),
235+
plain_list_type));
236+
237+
auto expected_array = table->column(0)->chunk(0);
238+
239+
ASSERT_OK_AND_ASSIGN(
240+
auto expected_plain,
241+
::arrow::compute::Cast(expected_array, plain_list_type));
242+
243+
ASSERT_TRUE(read_array.Equals(expected_plain));
244+
} else {
245+
ASSERT_TRUE(read_table->Equals(*table));
246+
}
247+
}
248+
}
206249
}
207250

208251
TEST(StatisticsTest, TruncateOnlyHalfMinMax) {

‎cpp/src/parquet/column_writer.cc‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1394,22 +1394,22 @@ class TypedColumnWriterImpl : public ColumnWriterImpl,
13941394
MaybeCalculateValidityBits(AddIfNotNull(def_levels, offset), batch_size,
13951395
&batch_num_values, &batch_num_spaced_values,
13961396
&null_count);
1397-
const int64_t total_null_count = batch_size - batch_num_values;
1397+
const int64_t parquet_null_count = batch_size - batch_num_values;
13981398

13991399
WriteLevelsSpaced(batch_size, AddIfNotNull(def_levels, offset),
14001400
AddIfNotNull(rep_levels, offset));
14011401
if (bits_buffer_ != nullptr) {
14021402
WriteValuesSpaced(AddIfNotNull(values, value_offset), batch_num_values,
14031403
batch_num_spaced_values, bits_buffer_->data(), /*offset=*/0,
1404-
/*num_levels=*/batch_size, total_null_count);
1404+
/*num_levels=*/batch_size, parquet_null_count);
14051405
} else {
14061406
WriteValuesSpaced(AddIfNotNull(values, value_offset), batch_num_values,
14071407
batch_num_spaced_values, valid_bits,
14081408
valid_bits_offset + value_offset, /*num_levels=*/batch_size,
1409-
total_null_count);
1409+
parquet_null_count);
14101410
}
14111411
CommitWriteAndCheckPageLimit(batch_size, batch_num_spaced_values,
1412-
total_null_count, check_page);
1412+
parquet_null_count, check_page);
14131413
value_offset += batch_num_spaced_values;
14141414

14151415
// Dictionary size checked separately from data page size since we
@@ -2040,7 +2040,7 @@ Status TypedColumnWriterImpl<ParquetType>::WriteArrowDictionary(
20402040
// had so we need to recompute it from def levels.
20412041
MaybeCalculateValidityBits(AddIfNotNull(def_levels, offset), batch_size,
20422042
&batch_num_values, &batch_num_spaced_values, &null_count);
2043-
const int64_t total_null_count = batch_size - batch_num_values;
2043+
const int64_t parquet_null_count = batch_size - batch_num_values;
20442044
WriteLevelsSpaced(batch_size, AddIfNotNull(def_levels, offset),
20452045
AddIfNotNull(rep_levels, offset));
20462046
std::shared_ptr<Array> writeable_indices =
@@ -2054,7 +2054,7 @@ Status TypedColumnWriterImpl<ParquetType>::WriteArrowDictionary(
20542054
dict_encoder->PutIndices(*writeable_indices);
20552055
// Update unencoded byte array data size to size statistics
20562056
UpdateUnencodedDataBytes();
2057-
CommitWriteAndCheckPageLimit(batch_size, batch_num_values, total_null_count,
2057+
CommitWriteAndCheckPageLimit(batch_size, batch_num_values, parquet_null_count,
20582058
check_page);
20592059
value_offset += batch_num_spaced_values;
20602060
};

0 commit comments

Comments
 (0)