@@ -1108,6 +1108,43 @@ void write_nested_int96_parquet_file(const std::string& file_path, int64_t times
11081108 ::parquet::WriterProperties::Builder ().build(), arrow_builder.build()));
11091109}
11101110
1111+ // Writes `id` (field id 1) and `outer` STRUCT (10) holding an ID-less `inner` STRUCT whose `ts` (30)
1112+ // is an INT96 timestamp: the ID-less wrapper Iceberg's Parquet projection keeps for a selected
1113+ // nested ID, holding a timestamp the way Spark writes it.
1114+ void write_idless_wrapper_int96_parquet_file (const std::string& file_path,
1115+ const std::vector<int64_t >& timestamps,
1116+ const std::vector<int32_t >& ids) {
1117+ ASSERT_EQ (timestamps.size (), ids.size ());
1118+ const auto field_id = [](int32_t id) {
1119+ return arrow::key_value_metadata ({" PARQUET:field_id" }, {std::to_string (id)});
1120+ };
1121+ const auto timestamp_type = arrow::timestamp (arrow::TimeUnit::MICRO , " UTC" );
1122+ arrow::TimestampBuilder timestamp_builder (timestamp_type, arrow::default_memory_pool ());
1123+ ASSERT_TRUE (timestamp_builder.AppendValues (timestamps).ok ());
1124+ auto timestamp_result = timestamp_builder.Finish ();
1125+ ASSERT_TRUE (timestamp_result.ok ()) << timestamp_result.status ();
1126+ auto ts_field = arrow::field (" ts" , timestamp_type, false )->WithMetadata (field_id (30 ));
1127+ auto inner_result = arrow::StructArray::Make ({*timestamp_result}, {ts_field});
1128+ ASSERT_TRUE (inner_result.ok ()) << inner_result.status ();
1129+ auto inner_field = arrow::field (" inner" , arrow::struct_ ({ts_field}), false );
1130+ auto outer_result = arrow::StructArray::Make ({*inner_result}, {inner_field});
1131+ ASSERT_TRUE (outer_result.ok ()) << outer_result.status ();
1132+ auto schema = arrow::schema ({
1133+ arrow::field (" id" , arrow::int32 (), false )->WithMetadata (field_id (1 )),
1134+ arrow::field (" outer" , arrow::struct_ ({inner_field}), false )->WithMetadata (field_id (10 )),
1135+ });
1136+ auto table = arrow::Table::Make (schema, {build_int32_array (ids), *outer_result});
1137+
1138+ auto file_result = arrow::io::FileOutputStream::Open (file_path);
1139+ ASSERT_TRUE (file_result.ok ()) << file_result.status ();
1140+ std::shared_ptr<arrow::io::FileOutputStream> out = *file_result;
1141+ ::parquet::ArrowWriterProperties::Builder arrow_builder;
1142+ arrow_builder.enable_force_write_int96_timestamps ();
1143+ PARQUET_THROW_NOT_OK (::parquet::arrow::WriteTable (
1144+ *table, arrow::default_memory_pool (), out, static_cast <int64_t >(ids.size ()),
1145+ ::parquet::WriterProperties::Builder ().build(), arrow_builder.build()));
1146+ }
1147+
11111148void write_iceberg_equality_delete_bigint_parquet_file (const std::string& file_path,
11121149 int32_t field_id, int64_t value) {
11131150 const auto metadata =
@@ -6734,6 +6771,126 @@ TEST(IcebergV2ReaderTest, IcebergNestedInt96TimestampTzReadsInstantWithTableMark
67346771 std::filesystem::remove_all (test_dir);
67356772}
67366773
6774+ // TableColumnMapper resolves a nested field ID through an ID-less STRUCT wrapper below the root as
6775+ // well (ParquetRecursivelyRetainsIdlessWrapperForSelectedLeafId), so the table's timestamptz marker
6776+ // has to reach the leaf the same way. Otherwise the INT96 leaf decodes as a UTC wall clock that the
6777+ // projection reads in the session time zone, eight hours early in Asia/Shanghai, and an equality
6778+ // key through the wrapper compares that wall clock with the adjusted-UTC tombstone an Iceberg
6779+ // writer stores, so the tombstone misses its row.
6780+ // NOLINTNEXTLINE(readability-function-cognitive-complexity): read and equality-delete matrix.
6781+ TEST (IcebergV2ReaderTest, IcebergInt96TimestampTzUnderIdlessWrapperReadsInstantWithTableMarker) {
6782+ TimezoneUtils::load_timezones_to_cache ();
6783+ const auto test_dir = std::filesystem::temp_directory_path () /
6784+ " doris_iceberg_idless_wrapper_int96_timestamp_tz_test" ;
6785+ std::filesystem::remove_all (test_dir);
6786+ std::filesystem::create_directories (test_dir);
6787+ const auto file_path = (test_dir / " split.parquet" ).string ();
6788+ const auto delete_file_path = (test_dir / " equality-delete.parquet" ).string ();
6789+ // 2024-01-01 00:00:00.123456 UTC and one second later.
6790+ constexpr int64_t T1_MICROS = 1'704'067'200'123'456L ;
6791+ constexpr int64_t T2_MICROS = T1_MICROS + 1'000'000L ;
6792+ write_idless_wrapper_int96_parquet_file (file_path, {T1_MICROS , T2_MICROS }, {1 , 2 });
6793+ write_iceberg_timestamp_equality_delete_parquet_file (delete_file_path, 30 , T1_MICROS , " ts" ,
6794+ true );
6795+
6796+ const auto tz_type = std::make_shared<DataTypeTimeStampTz>(6 );
6797+ const auto nullable_tz_type = make_nullable (tz_type);
6798+ auto inner_type = std::make_shared<DataTypeStruct>(DataTypes {tz_type}, Strings {" ts" });
6799+ auto inner = make_table_column (20 , " inner" , inner_type);
6800+ inner.children = {make_table_column (30 , " ts" , tz_type)};
6801+ auto outer = make_table_column (
6802+ 10 , " outer" ,
6803+ std::make_shared<DataTypeStruct>(DataTypes {inner_type}, Strings {" inner" }));
6804+ outer.children = {inner};
6805+
6806+ struct Case {
6807+ bool with_delete;
6808+ bool project_outer;
6809+ std::vector<int32_t > ids;
6810+ std::vector<std::string> timestamps;
6811+ };
6812+ const std::vector<Case> cases = {
6813+ {false ,
6814+ true ,
6815+ {1 , 2 },
6816+ {" 2024-01-01 00:00:00.123456+00:00" , " 2024-01-01 00:00:01.123456+00:00" }},
6817+ {true , true , {2 }, {" 2024-01-01 00:00:01.123456+00:00" }},
6818+ {true , false , {2 }, {}},
6819+ };
6820+ for (const auto & c : cases) {
6821+ SCOPED_TRACE (" with_delete=" + std::to_string (c.with_delete ) +
6822+ " project_outer=" + std::to_string (c.project_outer ));
6823+ std::vector<ColumnDefinition> projected_columns {
6824+ make_table_column (1 , " id" , std::make_shared<DataTypeInt32>())};
6825+ if (c.project_outer ) {
6826+ projected_columns.push_back (outer);
6827+ }
6828+ auto ts_field =
6829+ external_schema_field (" ts" , 30 , {}, std::nullopt ,
6830+ external_primitive_type (TPrimitiveType::TIMESTAMPTZ , -1 , 6 ));
6831+ ts_field.field_ptr ->__set_timestamp_is_adjusted_to_utc (true );
6832+ auto inner_field = external_struct_schema_field (" inner" , 20 , {std::move (ts_field)});
6833+ auto scan_params = make_local_parquet_scan_params ();
6834+ scan_params.__set_iceberg_scan_semantics_version (ICEBERG_SCAN_SEMANTICS_VERSION_2 );
6835+ scan_params.__set_parquet_timestamp_semantics_version (1 );
6836+ scan_params.__set_enable_mapping_timestamp_tz (true );
6837+ scan_params.__set_current_schema_id (100 );
6838+ scan_params.__set_history_schema_info ({external_schema (
6839+ 100 , {external_schema_field (" id" , 1 ),
6840+ external_struct_schema_field (" outer" , 10 , {std::move (inner_field)})})});
6841+
6842+ RuntimeProfile profile (" test_profile" );
6843+ RuntimeState state {TQueryOptions (), TQueryGlobals ()};
6844+ state.set_timezone (" Asia/Shanghai" );
6845+ io::FileReaderStats file_reader_stats;
6846+ io::FileCacheStatistics file_cache_stats;
6847+ auto io_ctx = make_io_context (&file_reader_stats, &file_cache_stats);
6848+ ShardedKVCache cache (1 );
6849+ doris::format::iceberg::IcebergTableReader reader;
6850+ init_iceberg_reader (&reader, projected_columns, &scan_params, io_ctx, &state, &profile);
6851+ std::vector<TIcebergDeleteFileDesc> delete_files;
6852+ if (c.with_delete ) {
6853+ delete_files.push_back (make_iceberg_equality_delete_file (delete_file_path, {30 }));
6854+ }
6855+ auto split_options = build_split_options (file_path);
6856+ split_options.cache = &cache;
6857+ split_options.current_range .__set_table_format_params (
6858+ make_iceberg_table_format_desc (file_path, delete_files));
6859+ ASSERT_TRUE (reader.prepare_split (split_options).ok ());
6860+
6861+ std::vector<int32_t > ids;
6862+ std::vector<std::string> timestamps;
6863+ bool eos = false ;
6864+ while (!eos) {
6865+ Block block = build_table_block (projected_columns);
6866+ const auto status = reader.get_block (&block, &eos);
6867+ ASSERT_TRUE (status.ok ()) << status;
6868+ if (block.rows () == 0 ) {
6869+ continue ;
6870+ }
6871+ const auto & id_column =
6872+ assert_cast<const ColumnInt32&>(expect_not_null_table_column (block, 0 ));
6873+ for (size_t row = 0 ; row < block.rows (); ++row) {
6874+ ids.push_back (id_column.get_element (row));
6875+ }
6876+ if (!c.project_outer ) {
6877+ continue ;
6878+ }
6879+ const auto & outer_column =
6880+ assert_cast<const ColumnStruct&>(expect_not_null_table_column (block, 1 ));
6881+ const auto & inner_column = assert_cast<const ColumnStruct&>(
6882+ expect_not_null_nullable_nested_column (outer_column.get_column (0 )));
6883+ for (size_t row = 0 ; row < block.rows (); ++row) {
6884+ timestamps.push_back (nullable_tz_type->to_string (inner_column.get_column (0 ), row));
6885+ }
6886+ }
6887+ EXPECT_EQ (ids, c.ids );
6888+ EXPECT_EQ (timestamps, c.timestamps );
6889+ ASSERT_TRUE (reader.close ().ok ());
6890+ }
6891+ std::filesystem::remove_all (test_dir);
6892+ }
6893+
67376894// A Spark INT96 key and the adjusted-UTC tombstone an Iceberg writer stores for the same
67386895// timestamptz instant decode to the same TIMESTAMPTZ once FE marks the field, in both the data file
67396896// and the delete file. Under timestamp-semantics version 1 the data key used to read as a UTC wall
0 commit comments