Skip to content

Commit 6c9fe7a

Browse files
committed
apacheGH-51097: [C++][Parquet] Cover nested null counts across page versions
Signed-off-by: Gang Wu <[email protected]>
1 parent ea6e4b1 commit 6c9fe7a

2 files changed

Lines changed: 41 additions & 27 deletions

File tree

‎cpp/src/parquet/arrow/arrow_statistics_test.cc‎

Lines changed: 39 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@
1818
#include "gtest/gtest.h"
1919

2020
#include "arrow/array.h"
21-
#include "arrow/compute/api.h"
2221
#include "arrow/array/builder_primitive.h"
2322
#include "arrow/array/builder_time.h"
23+
#include "arrow/compute/api.h"
2424
#include "arrow/table.h"
2525
#include "arrow/testing/gtest_util.h"
2626

@@ -171,34 +171,35 @@ TEST(StatisticsTest, FixedWidthLeafUnderListStructNullCount) {
171171
<< "data_page_version=" << static_cast<int>(data_page_version)
172172
<< ", use_dictionary=" << use_dictionary);
173173

174-
auto string_type =
175-
use_dictionary ? ::arrow::dictionary(::arrow::int32(), ::arrow::utf8())
176-
: ::arrow::utf8();
174+
auto string_type = use_dictionary
175+
? ::arrow::dictionary(::arrow::int32(), ::arrow::utf8())
176+
: ::arrow::utf8();
177177
auto list_type = ::arrow::list(::arrow::struct_(
178-
{::arrow::field("s", string_type),
179-
::arrow::field("i32", ::arrow::int32())}));
178+
{::arrow::field("s", string_type), ::arrow::field("i32", ::arrow::int32())}));
180179
auto schema = ::arrow::schema({::arrow::field("col", list_type)});
181-
182-
auto table = Table::Make(
183-
schema, {ArrayFromJSON(
184-
list_type,
185-
R"([[{"s":"a","i32":1}],null,[],[{"s":null,"i32":null},{"s":"b","i32":2}]])")});
180+
auto table = ::arrow::TableFromJSON(schema, {R"([
181+
[[{"s":"a","i32":1}]],
182+
[null],
183+
[[]],
184+
[[{"s":null,"i32":null},{"s":"b","i32":2}]]
185+
])"});
186186

187187
WriterProperties::Builder properties_builder;
188188
properties_builder.data_page_version(data_page_version);
189189
if (use_dictionary) {
190190
properties_builder.enable_dictionary();
191+
} else {
192+
properties_builder.disable_dictionary();
191193
}
192194

193195
std::shared_ptr<::arrow::ResizableBuffer> serialized_data = AllocateBuffer();
194196
auto out_stream =
195197
std::make_shared<::arrow::io::BufferOutputStream>(serialized_data);
196198

197-
ASSERT_OK_AND_ASSIGN(
198-
std::unique_ptr<FileWriter> writer,
199-
FileWriter::Open(*schema, default_memory_pool(), out_stream,
200-
properties_builder.build(),
201-
default_arrow_writer_properties()));
199+
ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileWriter> writer,
200+
FileWriter::Open(*schema, default_memory_pool(), out_stream,
201+
properties_builder.build(),
202+
default_arrow_writer_properties()));
202203
ASSERT_OK(writer->WriteTable(*table));
203204
ASSERT_OK(writer->Close());
204205
ASSERT_OK(out_stream->Close());
@@ -215,6 +216,22 @@ TEST(StatisticsTest, FixedWidthLeafUnderListStructNullCount) {
215216
ASSERT_NE(stats, nullptr);
216217
EXPECT_EQ(stats->null_count(), 3);
217218
EXPECT_EQ(stats->num_values(), 2);
219+
220+
if (data_page_version == ParquetDataPageVersion::V2) {
221+
auto page_reader = parquet_reader->RowGroup(0)->GetColumnPageReader(i);
222+
if (use_dictionary) {
223+
auto dictionary_page = page_reader->NextPage();
224+
ASSERT_NE(dictionary_page, nullptr);
225+
ASSERT_EQ(dictionary_page->type(), PageType::DICTIONARY_PAGE);
226+
}
227+
auto page = page_reader->NextPage();
228+
ASSERT_NE(page, nullptr);
229+
ASSERT_EQ(page->type(), PageType::DATA_PAGE_V2);
230+
auto data_page = std::static_pointer_cast<DataPageV2>(page);
231+
EXPECT_EQ(data_page->num_values(), 5);
232+
EXPECT_EQ(data_page->num_nulls(), 3);
233+
EXPECT_EQ(page_reader->NextPage(), nullptr);
234+
}
218235
}
219236

220237
ASSERT_OK_AND_ASSIGN(
@@ -225,20 +242,18 @@ TEST(StatisticsTest, FixedWidthLeafUnderListStructNullCount) {
225242
ASSERT_OK_AND_ASSIGN(auto read_table, file_reader->ReadTable());
226243

227244
if (use_dictionary) {
228-
auto plain_list_type = ::arrow::list(::arrow::struct_(
229-
{::arrow::field("s", ::arrow::utf8()),
230-
::arrow::field("i32", ::arrow::int32())}));
245+
auto plain_list_type =
246+
::arrow::list(::arrow::struct_({::arrow::field("s", ::arrow::utf8()),
247+
::arrow::field("i32", ::arrow::int32())}));
231248

232249
ASSERT_OK_AND_ASSIGN(
233250
auto read_array,
234-
::arrow::compute::Cast(read_table->column(0)->chunk(0),
235-
plain_list_type));
251+
::arrow::compute::Cast(read_table->column(0)->chunk(0), plain_list_type));
236252

237253
auto expected_array = table->column(0)->chunk(0);
238254

239-
ASSERT_OK_AND_ASSIGN(
240-
auto expected_plain,
241-
::arrow::compute::Cast(expected_array, plain_list_type));
255+
ASSERT_OK_AND_ASSIGN(auto expected_plain,
256+
::arrow::compute::Cast(expected_array, plain_list_type));
242257

243258
ASSERT_TRUE(read_array.Equals(expected_plain));
244259
} else {

‎cpp/src/parquet/column_writer.cc‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1394,10 +1394,10 @@ class TypedColumnWriterImpl : public ColumnWriterImpl,
13941394
MaybeCalculateValidityBits(AddIfNotNull(def_levels, offset), batch_size,
13951395
&batch_num_values, &batch_num_spaced_values,
13961396
&null_count);
1397-
const int64_t parquet_null_count = batch_size - batch_num_values;
13981397

13991398
WriteLevelsSpaced(batch_size, AddIfNotNull(def_levels, offset),
14001399
AddIfNotNull(rep_levels, offset));
1400+
const int64_t parquet_null_count = batch_size - batch_num_values;
14011401
if (bits_buffer_ != nullptr) {
14021402
WriteValuesSpaced(AddIfNotNull(values, value_offset), batch_num_values,
14031403
batch_num_spaced_values, bits_buffer_->data(), /*offset=*/0,
@@ -1751,7 +1751,6 @@ class TypedColumnWriterImpl : public ColumnWriterImpl,
17511751
internal::DefLevelsToBitmap(def_levels, batch_size, level_info_, &io);
17521752
*out_values_to_write = io.values_read - io.null_count;
17531753
*out_spaced_values_to_write = io.values_read;
1754-
// io.null_count excludes nulls from repeated ancestors.
17551754
*null_count = io.null_count;
17561755
}
17571756

@@ -2040,7 +2039,6 @@ Status TypedColumnWriterImpl<ParquetType>::WriteArrowDictionary(
20402039
// had so we need to recompute it from def levels.
20412040
MaybeCalculateValidityBits(AddIfNotNull(def_levels, offset), batch_size,
20422041
&batch_num_values, &batch_num_spaced_values, &null_count);
2043-
const int64_t parquet_null_count = batch_size - batch_num_values;
20442042
WriteLevelsSpaced(batch_size, AddIfNotNull(def_levels, offset),
20452043
AddIfNotNull(rep_levels, offset));
20462044
std::shared_ptr<Array> writeable_indices =
@@ -2054,6 +2052,7 @@ Status TypedColumnWriterImpl<ParquetType>::WriteArrowDictionary(
20542052
dict_encoder->PutIndices(*writeable_indices);
20552053
// Update unencoded byte array data size to size statistics
20562054
UpdateUnencodedDataBytes();
2055+
const int64_t parquet_null_count = batch_size - batch_num_values;
20572056
CommitWriteAndCheckPageLimit(batch_size, batch_num_values, parquet_null_count,
20582057
check_page);
20592058
value_offset += batch_num_spaced_values;

0 commit comments

Comments
 (0)