|
22 | 22 | #include <gtest/gtest.h> |
23 | 23 | #include <parquet/api/reader.h> |
24 | 24 | #include <parquet/arrow/reader.h> |
| 25 | +#include <parquet/page_index.h> |
25 | 26 | #include <parquet/schema.h> |
26 | 27 |
|
27 | 28 | #include <algorithm> |
@@ -178,11 +179,236 @@ class VParquetWriterTest : public testing::Test { |
178 | 179 | EXPECT_EQ(offsets, repeated); |
179 | 180 | } |
180 | 181 |
|
| 182 | + Block page_index_block() { |
| 183 | + auto type = std::make_shared<DataTypeInt32>(); |
| 184 | + auto nullable = make_nullable(type); |
| 185 | + auto ids = type->create_column(); |
| 186 | + auto values = nullable->create_column(); |
| 187 | + auto nulls = nullable->create_column(); |
| 188 | + for (int32_t row = 0; row < 12000; ++row) { |
| 189 | + ids->insert(Field::create_field<TYPE_INT>(row)); |
| 190 | + if (row % 7 == 0) { |
| 191 | + values->insert_default(); |
| 192 | + } else { |
| 193 | + values->insert(Field::create_field<TYPE_INT>(row % 64)); |
| 194 | + } |
| 195 | + nulls->insert_default(); |
| 196 | + } |
| 197 | + return Block({{std::move(ids), type, "id"}, |
| 198 | + {std::move(values), nullable, "value"}, |
| 199 | + {std::move(nulls), nullable, "all_null"}}); |
| 200 | + } |
| 201 | + |
| 202 | + void check_page_indexes(bool enabled) { |
| 203 | + auto reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); |
| 204 | + ASSERT_EQ(reader->metadata()->num_row_groups(), 1); |
| 205 | + auto group = reader->metadata()->RowGroup(0); |
| 206 | + ASSERT_EQ(group->num_rows(), 12000); |
| 207 | + for (int column = 0; column < group->num_columns(); ++column) { |
| 208 | + auto chunk = group->ColumnChunk(column); |
| 209 | + EXPECT_EQ(chunk->GetColumnIndexLocation().has_value(), enabled); |
| 210 | + EXPECT_EQ(chunk->GetOffsetIndexLocation().has_value(), enabled); |
| 211 | + if (!enabled) { |
| 212 | + continue; |
| 213 | + } |
| 214 | + auto index_reader = reader->GetPageIndexReader(); |
| 215 | + ASSERT_NE(index_reader, nullptr); |
| 216 | + auto indexes = index_reader->RowGroup(0); |
| 217 | + ASSERT_NE(indexes, nullptr); |
| 218 | + auto column_index = indexes->GetColumnIndex(column); |
| 219 | + auto offset_index = indexes->GetOffsetIndex(column); |
| 220 | + ASSERT_NE(column_index, nullptr); |
| 221 | + ASSERT_NE(offset_index, nullptr); |
| 222 | + const auto& pages = offset_index->page_locations(); |
| 223 | + ASSERT_FALSE(pages.empty()); |
| 224 | + EXPECT_EQ(pages.size(), column_index->null_pages().size()); |
| 225 | + EXPECT_EQ(pages.front().offset, chunk->data_page_offset()); |
| 226 | + EXPECT_EQ(pages.front().first_row_index, 0); |
| 227 | + const auto start = chunk->has_dictionary_page() ? chunk->dictionary_page_offset() |
| 228 | + : chunk->data_page_offset(); |
| 229 | + int64_t previous_row = -1; |
| 230 | + int64_t previous_end = start; |
| 231 | + for (const auto& page : pages) { |
| 232 | + EXPECT_GT(page.first_row_index, previous_row); |
| 233 | + EXPECT_LT(page.first_row_index, group->num_rows()); |
| 234 | + EXPECT_GE(page.offset, previous_end); |
| 235 | + EXPECT_GT(page.compressed_page_size, 0); |
| 236 | + EXPECT_LE(page.offset + page.compressed_page_size, |
| 237 | + start + chunk->total_compressed_size()); |
| 238 | + previous_row = page.first_row_index; |
| 239 | + previous_end = page.offset + page.compressed_page_size; |
| 240 | + } |
| 241 | + if (column == 0) { |
| 242 | + EXPECT_GT(pages.size(), 1); |
| 243 | + } else if (column == 2) { |
| 244 | + EXPECT_TRUE(std::all_of(column_index->null_pages().begin(), |
| 245 | + column_index->null_pages().end(), |
| 246 | + [](bool value) { return value; })); |
| 247 | + } |
| 248 | + } |
| 249 | + auto input = arrow::io::ReadableFile::Open(_file_path); |
| 250 | + ASSERT_TRUE(input.ok()) << input.status(); |
| 251 | + auto arrow_reader = ::parquet::arrow::OpenFile(*input, arrow::default_memory_pool()); |
| 252 | + ASSERT_TRUE(arrow_reader.ok()) << arrow_reader.status(); |
| 253 | + auto table = (*arrow_reader)->ReadTable(); |
| 254 | + ASSERT_TRUE(table.ok()) << table.status(); |
| 255 | + EXPECT_EQ((*table)->num_rows(), 12000); |
| 256 | + EXPECT_EQ((*table)->column(1)->null_count(), (12000 + 6) / 7); |
| 257 | + EXPECT_EQ((*table)->column(2)->null_count(), 12000); |
| 258 | + const auto& ids = assert_cast<const arrow::Int32Array&>(*(*table)->column(0)->chunk(0)); |
| 259 | + EXPECT_EQ(ids.Value(0), 0); |
| 260 | + EXPECT_EQ(ids.Value(11999), 11999); |
| 261 | + } |
| 262 | + |
181 | 263 | arrow::MemoryPool* _previous_pool = nullptr; |
182 | 264 | std::string _file_path; |
183 | 265 | std::shared_ptr<io::FileSystem> _fs; |
184 | 266 | }; |
185 | 267 |
|
| 268 | +TEST_F(VParquetWriterTest, ParquetPageIndexToggle) { |
| 269 | + for (const bool enabled : {false, true}) { |
| 270 | + for (const bool dictionary : {false, true}) { |
| 271 | + for (const auto version : |
| 272 | + {TParquetVersion::PARQUET_1_0, TParquetVersion::PARQUET_2_LATEST}) { |
| 273 | + TQueryOptions query_options; |
| 274 | + query_options.__set_enable_parquet_write_page_index(enabled); |
| 275 | + query_options.__set_parquet_write_data_page_size(1024); |
| 276 | + RuntimeState state(query_options, TQueryGlobals()); |
| 277 | + state.set_timezone("UTC"); |
| 278 | + auto block = page_index_block(); |
| 279 | + auto exprs = MockSlotRef::create_mock_contexts(block.get_data_types()); |
| 280 | + io::FileWriterPtr file; |
| 281 | + ASSERT_TRUE(_fs->create_file(_file_path, &file).ok()); |
| 282 | + ParquetFileOptions options {.compression_type = TParquetCompressionType::SNAPPY, |
| 283 | + .parquet_version = version, |
| 284 | + .parquet_disable_dictionary = !dictionary}; |
| 285 | + VParquetWriter writer(&state, file.get(), exprs, |
| 286 | + std::vector<std::string> {"id", "value", "all_null"}, false, |
| 287 | + options); |
| 288 | + ASSERT_TRUE(writer.open().ok()); |
| 289 | + ASSERT_TRUE(writer.write(block).ok()); |
| 290 | + ASSERT_TRUE(writer.close().ok()); |
| 291 | + check_page_indexes(enabled); |
| 292 | + } |
| 293 | + } |
| 294 | + } |
| 295 | +} |
| 296 | + |
| 297 | +TEST_F(VParquetWriterTest, ParquetPageIndexDataPageVersions) { |
| 298 | + for (const auto page_version : |
| 299 | + {::parquet::ParquetDataPageVersion::V1, ::parquet::ParquetDataPageVersion::V2}) { |
| 300 | + TQueryOptions query_options; |
| 301 | + query_options.__set_enable_parquet_write_page_index(true); |
| 302 | + query_options.__set_parquet_write_data_page_size(1024); |
| 303 | + RuntimeState state(query_options, TQueryGlobals()); |
| 304 | + auto block = page_index_block(); |
| 305 | + auto exprs = MockSlotRef::create_mock_contexts(block.get_data_types()); |
| 306 | + io::FileWriterPtr file; |
| 307 | + ASSERT_TRUE(_fs->create_file(_file_path, &file).ok()); |
| 308 | + const std::vector<std::string> names {"id", "value", "all_null"}; |
| 309 | + ParquetFileOptions options {.compression_type = TParquetCompressionType::SNAPPY, |
| 310 | + .parquet_version = TParquetVersion::PARQUET_2_LATEST}; |
| 311 | + VParquetWriter writer(&state, file.get(), exprs, names, false, options); |
| 312 | + ASSERT_TRUE(writer._parse_properties().ok()); |
| 313 | + // Exercise both page layouts with the production writer properties; this does not add |
| 314 | + // a SQL option for changing the existing data-page version. |
| 315 | + auto properties = ::parquet::WriterProperties::Builder(*writer._parquet_writer_properties) |
| 316 | + .data_page_version(page_version) |
| 317 | + ->build(); |
| 318 | + auto converter = writer._create_arrow_block_convertor(block.get_data_types(), names, "UTC", |
| 319 | + cctz::utc_time_zone(), false); |
| 320 | + ASSERT_TRUE(converter->init().ok()); |
| 321 | + std::shared_ptr<arrow::RecordBatch> batch; |
| 322 | + ASSERT_TRUE(converter->convert_to_arrow(block, arrow::default_memory_pool(), &batch).ok()); |
| 323 | + auto result = ::parquet::arrow::FileWriter::Open( |
| 324 | + *batch->schema(), arrow::default_memory_pool(), writer._outstream, properties, |
| 325 | + writer._arrow_properties); |
| 326 | + ASSERT_TRUE(result.ok()) << result.status(); |
| 327 | + auto arrow_writer = std::move(result).ValueOrDie(); |
| 328 | + ASSERT_TRUE(arrow_writer->WriteRecordBatch(*batch).ok()); |
| 329 | + ASSERT_TRUE(arrow_writer->Close().ok()); |
| 330 | + ASSERT_TRUE(writer._outstream->Close().ok()); |
| 331 | + check_page_indexes(true); |
| 332 | + } |
| 333 | +} |
| 334 | + |
| 335 | +TEST_F(VParquetWriterTest, ParquetPageSizePreservesHadoopLz4Limit) { |
| 336 | + for (const auto codec : {TParquetCompressionType::LZ4_HADOOP, TParquetCompressionType::LZ4}) { |
| 337 | + for (const int32_t requested_size : {65536, 262144}) { |
| 338 | + TQueryOptions query_options; |
| 339 | + query_options.__set_parquet_write_data_page_size(requested_size); |
| 340 | + RuntimeState state(query_options, TQueryGlobals()); |
| 341 | + state.set_timezone("UTC"); |
| 342 | + auto block = page_index_block(); |
| 343 | + auto exprs = MockSlotRef::create_mock_contexts(block.get_data_types()); |
| 344 | + io::FileWriterPtr file; |
| 345 | + ASSERT_TRUE(_fs->create_file(_file_path, &file).ok()); |
| 346 | + ParquetFileOptions options {.compression_type = codec, |
| 347 | + .parquet_version = TParquetVersion::PARQUET_1_0}; |
| 348 | + VParquetWriter writer(&state, file.get(), exprs, |
| 349 | + std::vector<std::string> {"id", "value", "all_null"}, false, |
| 350 | + options); |
| 351 | + ASSERT_TRUE(writer.open().ok()); |
| 352 | + EXPECT_TRUE(writer._parquet_writer_properties->page_index_enabled()); |
| 353 | + EXPECT_EQ(writer._parquet_writer_properties->data_pagesize(), |
| 354 | + codec == TParquetCompressionType::LZ4_HADOOP |
| 355 | + ? std::min(requested_size, 131072) |
| 356 | + : requested_size); |
| 357 | + ASSERT_TRUE(writer.close().ok()); |
| 358 | + } |
| 359 | + } |
| 360 | +} |
| 361 | + |
| 362 | +TEST_F(VParquetWriterTest, HiveAndIcebergParquetPageIndexOptions) { |
| 363 | + const std::string json = |
| 364 | + R"({"type":"struct","fields":[{"id":1,"name":"id","required":true,"type":"int"},{"id":2,"name":"value","required":false,"type":"int"},{"id":3,"name":"all_null","required":false,"type":"int"}]})"; |
| 365 | + auto schema = iceberg::SchemaParser::from_json(json); |
| 366 | + const std::map<std::string, std::string> hadoop_conf; |
| 367 | + for (const bool iceberg_writer : {false, true}) { |
| 368 | + for (const bool enabled : {false, true}) { |
| 369 | + TQueryOptions query_options; |
| 370 | + query_options.__set_enable_parquet_write_page_index(enabled); |
| 371 | + query_options.__set_parquet_write_data_page_size(1024); |
| 372 | + RuntimeState state(query_options, TQueryGlobals()); |
| 373 | + state.set_timezone("UTC"); |
| 374 | + auto block = page_index_block(); |
| 375 | + auto exprs = MockSlotRef::create_mock_contexts(block.get_data_types()); |
| 376 | + const auto name = "page_index_" + UniqueId::gen_uid().to_string(); |
| 377 | + _file_path = "./" + name + "-0.parquet"; |
| 378 | + TDataSink sink; |
| 379 | + if (iceberg_writer) { |
| 380 | + VIcebergPartitionWriter writer(sink, {}, exprs, *schema, &json, |
| 381 | + {"id", "value", "all_null"}, |
| 382 | + {.write_path = ".", |
| 383 | + .original_write_path = ".", |
| 384 | + .target_path = ".", |
| 385 | + .file_type = TFileType::FILE_LOCAL, |
| 386 | + .broker_addresses = {}}, |
| 387 | + name, 0, TFileFormatType::FORMAT_PARQUET, |
| 388 | + TFileCompressType::PLAIN, hadoop_conf); |
| 389 | + ASSERT_TRUE(writer.open(&state, nullptr, nullptr).ok()); |
| 390 | + ASSERT_TRUE(writer.write(block).ok()); |
| 391 | + ASSERT_TRUE(writer.close(Status::OK()).ok()); |
| 392 | + } else { |
| 393 | + VHivePartitionWriter writer(sink, "", TUpdateMode::APPEND, exprs, |
| 394 | + {"id", "value", "all_null"}, |
| 395 | + {.write_path = ".", |
| 396 | + .original_write_path = ".", |
| 397 | + .target_path = ".", |
| 398 | + .file_type = TFileType::FILE_LOCAL, |
| 399 | + .broker_addresses = {}}, |
| 400 | + name, 0, TFileFormatType::FORMAT_PARQUET, |
| 401 | + TFileCompressType::PLAIN, nullptr, hadoop_conf); |
| 402 | + ASSERT_TRUE(writer.open(&state, nullptr).ok()); |
| 403 | + ASSERT_TRUE(writer.write(block).ok()); |
| 404 | + ASSERT_TRUE(writer.close(Status::OK()).ok()); |
| 405 | + } |
| 406 | + check_page_indexes(enabled); |
| 407 | + ASSERT_TRUE(_fs->delete_file(_file_path).ok()); |
| 408 | + } |
| 409 | + } |
| 410 | +} |
| 411 | + |
186 | 412 | TEST_F(VParquetWriterTest, IcebergSplitOffsetsIncludeDictionaryPages) { |
187 | 413 | check_split_offsets(3, true); |
188 | 414 | } |
|
0 commit comments