|
42 | 42 | #include "exprs/vexpr_context.h" |
43 | 43 | #include "format/orc/vorc_reader.h" |
44 | 44 | #include "format/parquet/vparquet_reader.h" |
| 45 | +#include "format/table/iceberg_scan_semantics.h" |
45 | 46 | #include "gtest/gtest_pred_impl.h" |
46 | 47 | #include "io/fs/local_file_system.h" |
47 | 48 | #include "orc/sargs/SearchArgument.hh" |
@@ -322,4 +323,150 @@ TEST_F(ParquetReadLinesTest, test4) { |
322 | 323 | read_parquet_lines(numeric_types, types, read_lines, block_dump); |
323 | 324 | } |
324 | 325 |
|
| 326 | +TEST_F(ParquetReadLinesTest, iceberg_row_id_fetch_materializes_readded_missing_column) { |
| 327 | + TDescriptorTable thrift_desc; |
| 328 | + TTableDescriptor table_desc; |
| 329 | + table_desc.__set_id(0); |
| 330 | + table_desc.__set_tableType(TTableType::OLAP_TABLE); |
| 331 | + table_desc.__set_numCols(0); |
| 332 | + table_desc.__set_numClusteringCols(0); |
| 333 | + thrift_desc.tableDescriptors.push_back(table_desc); |
| 334 | + thrift_desc.__isset.tableDescriptors = true; |
| 335 | + |
| 336 | + const std::vector<std::tuple<std::string, int32_t, TPrimitiveType::type>> columns { |
| 337 | + {"new_name", 2, TPrimitiveType::STRING}, |
| 338 | + {"data", 3, TPrimitiveType::STRING}, |
| 339 | + {"id", 4, TPrimitiveType::INT}, |
| 340 | + }; |
| 341 | + for (size_t index = 0; index < columns.size(); ++index) { |
| 342 | + const auto& [name, field_id, primitive_type] = columns[index]; |
| 343 | + TTypeNode type_node; |
| 344 | + type_node.__set_type(TTypeNodeType::SCALAR); |
| 345 | + TScalarType scalar_type; |
| 346 | + scalar_type.__set_type(primitive_type); |
| 347 | + type_node.__set_scalar_type(scalar_type); |
| 348 | + TTypeDesc type; |
| 349 | + type.types.push_back(type_node); |
| 350 | + |
| 351 | + TSlotDescriptor slot; |
| 352 | + slot.__set_id(cast_set<int32_t>(index)); |
| 353 | + slot.__set_parent(0); |
| 354 | + slot.__set_slotType(type); |
| 355 | + slot.__set_columnPos(cast_set<int32_t>(index)); |
| 356 | + slot.__set_byteOffset(0); |
| 357 | + slot.__set_nullIndicatorByte(0); |
| 358 | + slot.__set_nullIndicatorBit(cast_set<int32_t>(index)); |
| 359 | + slot.__set_colName(name); |
| 360 | + slot.__set_slotIdx(cast_set<int32_t>(index)); |
| 361 | + slot.__set_isMaterialized(true); |
| 362 | + slot.__set_col_unique_id(field_id); |
| 363 | + thrift_desc.slotDescriptors.push_back(slot); |
| 364 | + } |
| 365 | + thrift_desc.__isset.slotDescriptors = true; |
| 366 | + |
| 367 | + TTupleDescriptor tuple; |
| 368 | + tuple.__set_id(0); |
| 369 | + tuple.__set_byteSize(16); |
| 370 | + tuple.__set_numNullBytes(1); |
| 371 | + tuple.__set_tableId(0); |
| 372 | + tuple.__isset.tableId = true; |
| 373 | + thrift_desc.tupleDescriptors.push_back(tuple); |
| 374 | + |
| 375 | + ObjectPool object_pool; |
| 376 | + DescriptorTbl* desc_tbl = nullptr; |
| 377 | + ASSERT_TRUE(DescriptorTbl::create(&object_pool, thrift_desc, &desc_tbl).ok()); |
| 378 | + auto* tuple_desc = const_cast<TupleDescriptor*>(desc_tbl->get_tuple_descriptor(0)); |
| 379 | + ASSERT_NE(tuple_desc, nullptr); |
| 380 | + |
| 381 | + const auto make_field = [](const std::string& name, int32_t field_id, |
| 382 | + TPrimitiveType::type primitive_type, bool optional) { |
| 383 | + auto field = std::make_shared<schema::external::TField>(); |
| 384 | + field->__set_name(name); |
| 385 | + field->__set_id(field_id); |
| 386 | + field->__set_is_optional(optional); |
| 387 | + TColumnType type; |
| 388 | + type.__set_type(primitive_type); |
| 389 | + field->__set_type(type); |
| 390 | + schema::external::TFieldPtr field_ptr; |
| 391 | + field_ptr.__set_field_ptr(std::move(field)); |
| 392 | + return field_ptr; |
| 393 | + }; |
| 394 | + schema::external::TStructField root; |
| 395 | + root.__set_fields({make_field("new_new_id", 1, TPrimitiveType::INT, false), |
| 396 | + make_field("new_name", 2, TPrimitiveType::STRING, true), |
| 397 | + make_field("data", 3, TPrimitiveType::STRING, false), |
| 398 | + make_field("id", 4, TPrimitiveType::INT, true)}); |
| 399 | + schema::external::TSchema schema; |
| 400 | + schema.__set_schema_id(4); |
| 401 | + schema.__set_root_field(root); |
| 402 | + |
| 403 | + TFileScanRangeParams scan_params; |
| 404 | + scan_params.__set_file_type(TFileType::FILE_LOCAL); |
| 405 | + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); |
| 406 | + scan_params.__set_num_of_columns_from_file(cast_set<int32_t>(columns.size())); |
| 407 | + scan_params.__set_current_schema_id(4); |
| 408 | + scan_params.__set_history_schema_info({schema}); |
| 409 | + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2); |
| 410 | + for (size_t index = 0; index < columns.size(); ++index) { |
| 411 | + const auto* slot = tuple_desc->slots()[index]; |
| 412 | + TFileScanSlotInfo slot_info; |
| 413 | + slot_info.__set_slot_id(slot->id()); |
| 414 | + slot_info.__set_is_file_slot(true); |
| 415 | + scan_params.required_slots.push_back(slot_info); |
| 416 | + scan_params.default_value_of_src_slot.emplace(slot->id(), TExpr {}); |
| 417 | + scan_params.column_idxs.push_back(cast_set<int32_t>(index + 1)); |
| 418 | + scan_params.slot_name_to_schema_pos.emplace(slot->col_name(), cast_set<int32_t>(index + 1)); |
| 419 | + } |
| 420 | + scan_params.__isset.required_slots = true; |
| 421 | + scan_params.__isset.default_value_of_src_slot = true; |
| 422 | + scan_params.__isset.column_idxs = true; |
| 423 | + scan_params.__isset.slot_name_to_schema_pos = true; |
| 424 | + |
| 425 | + const std::string path = |
| 426 | + "./docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/" |
| 427 | + "equality_delete_par_1/data/" |
| 428 | + "00000-0-bd4d0a30-cdf6-48d7-933d-91e860870eb9-00001.parquet"; |
| 429 | + io::FileReaderSPtr file_reader; |
| 430 | + ASSERT_TRUE(io::global_local_filesystem()->open_file(path, &file_reader).ok()); |
| 431 | + TFileRangeDesc range; |
| 432 | + range.__set_path(path); |
| 433 | + range.__set_start_offset(0); |
| 434 | + range.__set_size(file_reader->size()); |
| 435 | + range.__set_file_size(file_reader->size()); |
| 436 | + range.__set_format_type(TFileFormatType::FORMAT_PARQUET); |
| 437 | + TTableFormatFileDesc table_format; |
| 438 | + table_format.__set_table_format_type("iceberg"); |
| 439 | + table_format.__set_iceberg_params(TIcebergFileDesc {}); |
| 440 | + range.__set_table_format_params(table_format); |
| 441 | + |
| 442 | + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; |
| 443 | + runtime_state.set_desc_tbl(desc_tbl); |
| 444 | + std::unordered_map<std::string, int> colname_to_slot_id; |
| 445 | + Block block; |
| 446 | + for (const auto* slot : tuple_desc->slots()) { |
| 447 | + colname_to_slot_id.emplace(slot->col_name(), slot->id()); |
| 448 | + block.insert({slot->get_empty_mutable_column(), slot->type(), slot->col_name()}); |
| 449 | + } |
| 450 | + RuntimeProfile profile("ExternalRowIDFetcher"); |
| 451 | + auto scanner = FileScanner::create_unique(&runtime_state, &profile, &scan_params, |
| 452 | + &colname_to_slot_id, tuple_desc); |
| 453 | + ASSERT_TRUE(scanner->prepare_for_read_lines(range).ok()); |
| 454 | + ExternalFileMappingInfo external_info(0, range, false); |
| 455 | + int64_t init_reader_ms = 0; |
| 456 | + int64_t get_block_ms = 0; |
| 457 | + const auto status = scanner->read_lines_from_range(range, {0}, &block, external_info, |
| 458 | + &init_reader_ms, &get_block_ms); |
| 459 | + ASSERT_TRUE(status.ok()) << status; |
| 460 | + ASSERT_EQ(block.rows(), 1); |
| 461 | + EXPECT_EQ(block.get_by_position(block.get_position_by_name("new_name")) |
| 462 | + .column->get_data_at(0) |
| 463 | + .to_string(), |
| 464 | + "bob"); |
| 465 | + EXPECT_EQ(block.get_by_position(block.get_position_by_name("data")) |
| 466 | + .column->get_data_at(0) |
| 467 | + .to_string(), |
| 468 | + "e"); |
| 469 | + EXPECT_TRUE(block.get_by_position(block.get_position_by_name("id")).column->is_null_at(0)); |
| 470 | +} |
| 471 | + |
325 | 472 | } // namespace doris |
0 commit comments