|
18 | 18 | #include "gtest/gtest.h" |
19 | 19 |
|
20 | 20 | #include "arrow/array.h" |
| 21 | +#include "arrow/compute/api.h" |
21 | 22 | #include "arrow/array/builder_primitive.h" |
22 | 23 | #include "arrow/array/builder_time.h" |
23 | 24 | #include "arrow/table.h" |
@@ -161,48 +162,90 @@ INSTANTIATE_TEST_SUITE_P( |
161 | 162 | /*expected_max=*/"z"})); |
162 | 163 |
|
163 | 164 | TEST(StatisticsTest, FixedWidthLeafUnderListStructNullCount) { |
164 | | - // Null counts for fixed-width leaves under list<struct<...>> |
165 | | - // must include null and empty list entries from the repeated ancestor. |
166 | | - auto schema = ::arrow::schema({::arrow::field( |
167 | | - "col", ::arrow::list(::arrow::struct_( |
168 | | - {::arrow::field("s", ::arrow::utf8()), |
169 | | - ::arrow::field("i32", ::arrow::int32())})))}); |
170 | | - |
171 | | - auto table = ::arrow::Table::Make( |
172 | | - schema, |
173 | | - {::arrow::ArrayFromJSON( |
174 | | - ::arrow::list(::arrow::struct_( |
175 | | - {::arrow::field("s", ::arrow::utf8()), |
176 | | - ::arrow::field("i32", ::arrow::int32())})), |
177 | | - R"([[{"s":"a","i32":1}],null,[],[{"s":null,"i32":null},{"s":"b","i32":2}]])")}); |
178 | | - |
179 | | - std::shared_ptr<::arrow::ResizableBuffer> serialized_data = AllocateBuffer(); |
180 | | - auto out_stream = |
181 | | - std::make_shared<::arrow::io::BufferOutputStream>(serialized_data); |
182 | | - |
183 | | - ASSERT_OK_AND_ASSIGN( |
184 | | - std::unique_ptr<FileWriter> writer, |
185 | | - FileWriter::Open(*schema, default_memory_pool(), out_stream, |
186 | | - default_writer_properties(), |
187 | | - default_arrow_writer_properties())); |
188 | | - ASSERT_OK(writer->WriteTable(*table)); |
189 | | - ASSERT_OK(writer->Close()); |
190 | | - ASSERT_OK(out_stream->Close()); |
191 | | - |
192 | | - auto buffer_reader = std::make_shared<::arrow::io::BufferReader>(serialized_data); |
193 | | - auto parquet_reader = ParquetFileReader::Open(std::move(buffer_reader)); |
194 | | - auto metadata = parquet_reader->metadata(); |
195 | | - auto row_group = metadata->RowGroup(0); |
196 | | - |
197 | | - ASSERT_EQ(row_group->num_columns(), 2); |
198 | | - |
199 | | - auto int32_stats = row_group->ColumnChunk(1)->statistics(); |
200 | | - ASSERT_NE(int32_stats, nullptr); |
201 | | - |
202 | | - // Fixed-width leaves must include nulls from repeated ancestors |
203 | | - // (e.g. null or empty lists) in the column statistics. |
204 | | - EXPECT_EQ(int32_stats->null_count(), 3); |
205 | | - EXPECT_EQ(int32_stats->num_values(), 2); |
| 165 | + // Null counts for leaves under list<struct<...>> must include null and empty |
| 166 | + // list entries from the repeated ancestor. |
| 167 | + for (const auto data_page_version : |
| 168 | + {ParquetDataPageVersion::V1, ParquetDataPageVersion::V2}) { |
| 169 | + for (const bool use_dictionary : {false, true}) { |
| 170 | + SCOPED_TRACE(::testing::Message() |
| 171 | + << "data_page_version=" << static_cast<int>(data_page_version) |
| 172 | + << ", use_dictionary=" << use_dictionary); |
| 173 | + |
| 174 | + auto string_type = |
| 175 | + use_dictionary ? ::arrow::dictionary(::arrow::int32(), ::arrow::utf8()) |
| 176 | + : ::arrow::utf8(); |
| 177 | + auto list_type = ::arrow::list(::arrow::struct_( |
| 178 | + {::arrow::field("s", string_type), |
| 179 | + ::arrow::field("i32", ::arrow::int32())})); |
| 180 | + auto schema = ::arrow::schema({::arrow::field("col", list_type)}); |
| 181 | + |
| 182 | + auto table = Table::Make( |
| 183 | + schema, {ArrayFromJSON( |
| 184 | + list_type, |
| 185 | + R"([[{"s":"a","i32":1}],null,[],[{"s":null,"i32":null},{"s":"b","i32":2}]])")}); |
| 186 | + |
| 187 | + WriterProperties::Builder properties_builder; |
| 188 | + properties_builder.data_page_version(data_page_version); |
| 189 | + if (use_dictionary) { |
| 190 | + properties_builder.enable_dictionary(); |
| 191 | + } |
| 192 | + |
| 193 | + std::shared_ptr<::arrow::ResizableBuffer> serialized_data = AllocateBuffer(); |
| 194 | + auto out_stream = |
| 195 | + std::make_shared<::arrow::io::BufferOutputStream>(serialized_data); |
| 196 | + |
| 197 | + ASSERT_OK_AND_ASSIGN( |
| 198 | + std::unique_ptr<FileWriter> writer, |
| 199 | + FileWriter::Open(*schema, default_memory_pool(), out_stream, |
| 200 | + properties_builder.build(), |
| 201 | + default_arrow_writer_properties())); |
| 202 | + ASSERT_OK(writer->WriteTable(*table)); |
| 203 | + ASSERT_OK(writer->Close()); |
| 204 | + ASSERT_OK(out_stream->Close()); |
| 205 | + |
| 206 | + auto buffer_reader = std::make_shared<::arrow::io::BufferReader>(serialized_data); |
| 207 | + auto parquet_reader = ParquetFileReader::Open(std::move(buffer_reader)); |
| 208 | + auto metadata = parquet_reader->metadata(); |
| 209 | + auto row_group = metadata->RowGroup(0); |
| 210 | + |
| 211 | + ASSERT_EQ(row_group->num_columns(), 2); |
| 212 | + |
| 213 | + for (int i = 0; i < 2; ++i) { |
| 214 | + auto stats = row_group->ColumnChunk(i)->statistics(); |
| 215 | + ASSERT_NE(stats, nullptr); |
| 216 | + EXPECT_EQ(stats->null_count(), 3); |
| 217 | + EXPECT_EQ(stats->num_values(), 2); |
| 218 | + } |
| 219 | + |
| 220 | + ASSERT_OK_AND_ASSIGN( |
| 221 | + auto file_reader, |
| 222 | + FileReader::Make(default_memory_pool(), std::move(parquet_reader), |
| 223 | + default_arrow_reader_properties())); |
| 224 | + |
| 225 | + ASSERT_OK_AND_ASSIGN(auto read_table, file_reader->ReadTable()); |
| 226 | + |
| 227 | + if (use_dictionary) { |
| 228 | + auto plain_list_type = ::arrow::list(::arrow::struct_( |
| 229 | + {::arrow::field("s", ::arrow::utf8()), |
| 230 | + ::arrow::field("i32", ::arrow::int32())})); |
| 231 | + |
| 232 | + ASSERT_OK_AND_ASSIGN( |
| 233 | + auto read_array, |
| 234 | + ::arrow::compute::Cast(read_table->column(0)->chunk(0), |
| 235 | + plain_list_type)); |
| 236 | + |
| 237 | + auto expected_array = table->column(0)->chunk(0); |
| 238 | + |
| 239 | + ASSERT_OK_AND_ASSIGN( |
| 240 | + auto expected_plain, |
| 241 | + ::arrow::compute::Cast(expected_array, plain_list_type)); |
| 242 | + |
| 243 | + ASSERT_TRUE(read_array.Equals(expected_plain)); |
| 244 | + } else { |
| 245 | + ASSERT_TRUE(read_table->Equals(*table)); |
| 246 | + } |
| 247 | + } |
| 248 | + } |
206 | 249 | } |
207 | 250 |
|
208 | 251 | TEST(StatisticsTest, TruncateOnlyHalfMinMax) { |
|
0 commit comments