@@ -260,6 +260,19 @@ std::shared_ptr<arrow::Array> build_nullable_int64_array(
260260 return finish_array (&builder);
261261}
262262
263+ std::shared_ptr<arrow::Array> build_nullable_string_array (
264+ const std::vector<std::optional<std::string>>& values) {
265+ arrow::StringBuilder builder;
266+ for (const auto & value : values) {
267+ if (value.has_value ()) {
268+ EXPECT_TRUE (builder.Append (*value).ok ());
269+ } else {
270+ EXPECT_TRUE (builder.AppendNull ().ok ());
271+ }
272+ }
273+ return finish_array (&builder);
274+ }
275+
263276std::shared_ptr<arrow::Array> build_string_array (const std::vector<std::string>& values) {
264277 arrow::StringBuilder builder;
265278 for (const auto & value : values) {
@@ -405,6 +418,31 @@ void write_position_delete_parquet_file(const std::string& file_path,
405418 builder.build ()));
406419}
407420
421+ void write_nullable_position_delete_parquet_file (
422+ const std::string& file_path,
423+ const std::vector<std::optional<std::string>>& data_file_paths,
424+ const std::vector<std::optional<int64_t >>& positions) {
425+ ASSERT_EQ (data_file_paths.size (), positions.size ());
426+ auto schema = arrow::schema ({
427+ arrow::field (" file_path" , arrow::utf8 (), true ),
428+ arrow::field (" pos" , arrow::int64 (), true ),
429+ });
430+ auto table = arrow::Table::Make (schema, {build_nullable_string_array (data_file_paths),
431+ build_nullable_int64_array (positions)});
432+
433+ auto file_result = arrow::io::FileOutputStream::Open (file_path);
434+ ASSERT_TRUE (file_result.ok ()) << file_result.status ();
435+ std::shared_ptr<arrow::io::FileOutputStream> out = *file_result;
436+
437+ ::parquet::WriterProperties::Builder builder;
438+ builder.version (::parquet::ParquetVersion::PARQUET_2_6 );
439+ builder.data_page_version (::parquet::ParquetDataPageVersion::V2 );
440+ builder.compression (::parquet::Compression::UNCOMPRESSED );
441+ PARQUET_THROW_NOT_OK (::parquet::arrow::WriteTable (*table, arrow::default_memory_pool (), out,
442+ static_cast <int64_t >(positions.size ()),
443+ builder.build ()));
444+ }
445+
408446int64_t write_iceberg_deletion_vector_file (const std::string& file_path,
409447 const std::vector<uint64_t >& deleted_positions) {
410448 roaring::Roaring64Map rows;
@@ -1980,6 +2018,147 @@ TEST(IcebergV2ReaderTest, IcebergTableReaderAppliesPositionDeleteFile) {
19802018 std::filesystem::remove_all (test_dir);
19812019}
19822020
2021+ TEST (IcebergV2ReaderTest, IcebergTableReaderAppliesNullablePositionDeleteFileWithoutNulls) {
2022+ const auto test_dir = std::filesystem::temp_directory_path () /
2023+ " doris_iceberg_nullable_position_delete_file_test" ;
2024+ std::filesystem::remove_all (test_dir);
2025+ std::filesystem::create_directories (test_dir);
2026+
2027+ const auto file_path = (test_dir / " split.parquet" ).string ();
2028+ const auto delete_file_path = (test_dir / " position-delete.parquet" ).string ();
2029+ write_int_pair_parquet_file (file_path, {1 , 2 , 3 , 4 , 5 }, {10 , 20 , 30 , 40 , 50 },
2030+ {" one" , " two" , " three" , " four" , " five" });
2031+ write_nullable_position_delete_parquet_file (delete_file_path, {file_path, file_path},
2032+ {int64_t {1 }, int64_t {3 }});
2033+
2034+ std::vector<ColumnDefinition> projected_columns;
2035+ projected_columns.push_back (make_table_column (0 , " id" , std::make_shared<DataTypeInt32>()));
2036+
2037+ RuntimeProfile profile (" test_profile" );
2038+ RuntimeState state {TQueryOptions (), TQueryGlobals ()};
2039+ auto scan_params = make_local_parquet_scan_params ();
2040+ io::FileReaderStats file_reader_stats;
2041+ io::FileCacheStatistics file_cache_stats;
2042+ auto io_ctx = make_io_context (&file_reader_stats, &file_cache_stats);
2043+ ShardedKVCache cache (1 );
2044+ doris::format::iceberg::IcebergTableReader reader;
2045+ ASSERT_TRUE (reader.init ({
2046+ .projected_columns = projected_columns,
2047+ .conjuncts = {},
2048+ .format = FileFormat::PARQUET ,
2049+ .scan_params = &scan_params,
2050+ .io_ctx = io_ctx,
2051+ .runtime_state = &state,
2052+ .scanner_profile = &profile,
2053+ })
2054+ .ok ());
2055+
2056+ auto split_options = build_split_options (file_path);
2057+ split_options.cache = &cache;
2058+ split_options.current_range .__set_table_format_params (make_iceberg_table_format_desc (
2059+ file_path, {make_iceberg_position_delete_file (delete_file_path)}));
2060+ ASSERT_TRUE (reader.prepare_split (split_options).ok ());
2061+
2062+ EXPECT_EQ (read_iceberg_ids (&reader, projected_columns), std::vector<int32_t >({1 , 3 , 5 }));
2063+
2064+ ASSERT_TRUE (reader.close ().ok ());
2065+ std::filesystem::remove_all (test_dir);
2066+ }
2067+
2068+ TEST (IcebergV2ReaderTest, IcebergTableReaderRejectsNullablePositionDeleteFileWithActualNulls) {
2069+ const auto test_dir = std::filesystem::temp_directory_path () /
2070+ " doris_iceberg_nullable_position_delete_actual_null_test" ;
2071+ std::filesystem::remove_all (test_dir);
2072+ std::filesystem::create_directories (test_dir);
2073+
2074+ const auto file_path = (test_dir / " split.parquet" ).string ();
2075+ const auto delete_file_path = (test_dir / " position-delete.parquet" ).string ();
2076+ write_int_pair_parquet_file (file_path, {1 , 2 , 3 }, {10 , 20 , 30 }, {" one" , " two" , " three" });
2077+ write_nullable_position_delete_parquet_file (delete_file_path, {file_path, std::nullopt },
2078+ {int64_t {1 }, int64_t {2 }});
2079+
2080+ std::vector<ColumnDefinition> projected_columns;
2081+ projected_columns.push_back (make_table_column (0 , " id" , std::make_shared<DataTypeInt32>()));
2082+
2083+ RuntimeProfile profile (" test_profile" );
2084+ RuntimeState state {TQueryOptions (), TQueryGlobals ()};
2085+ auto scan_params = make_local_parquet_scan_params ();
2086+ io::FileReaderStats file_reader_stats;
2087+ io::FileCacheStatistics file_cache_stats;
2088+ auto io_ctx = make_io_context (&file_reader_stats, &file_cache_stats);
2089+ ShardedKVCache cache (1 );
2090+ doris::format::iceberg::IcebergTableReader reader;
2091+ ASSERT_TRUE (reader.init ({
2092+ .projected_columns = projected_columns,
2093+ .conjuncts = {},
2094+ .format = FileFormat::PARQUET ,
2095+ .scan_params = &scan_params,
2096+ .io_ctx = io_ctx,
2097+ .runtime_state = &state,
2098+ .scanner_profile = &profile,
2099+ })
2100+ .ok ());
2101+
2102+ auto split_options = build_split_options (file_path);
2103+ split_options.cache = &cache;
2104+ split_options.current_range .__set_table_format_params (make_iceberg_table_format_desc (
2105+ file_path, {make_iceberg_position_delete_file (delete_file_path)}));
2106+ auto status = reader.prepare_split (split_options);
2107+ ASSERT_FALSE (status.ok ());
2108+ EXPECT_NE (status.to_string ().find (" file_path contains null values" ), std::string::npos)
2109+ << status.to_string ();
2110+
2111+ ASSERT_TRUE (reader.close ().ok ());
2112+ std::filesystem::remove_all (test_dir);
2113+ }
2114+
2115+ TEST (IcebergV2ReaderTest, IcebergTableReaderRejectsNullablePositionDeletePosWithActualNulls) {
2116+ const auto test_dir = std::filesystem::temp_directory_path () /
2117+ " doris_iceberg_nullable_position_delete_pos_null_test" ;
2118+ std::filesystem::remove_all (test_dir);
2119+ std::filesystem::create_directories (test_dir);
2120+
2121+ const auto file_path = (test_dir / " split.parquet" ).string ();
2122+ const auto delete_file_path = (test_dir / " position-delete.parquet" ).string ();
2123+ write_int_pair_parquet_file (file_path, {1 , 2 , 3 }, {10 , 20 , 30 }, {" one" , " two" , " three" });
2124+ write_nullable_position_delete_parquet_file (delete_file_path, {file_path, file_path},
2125+ {int64_t {1 }, std::nullopt });
2126+
2127+ std::vector<ColumnDefinition> projected_columns;
2128+ projected_columns.push_back (make_table_column (0 , " id" , std::make_shared<DataTypeInt32>()));
2129+
2130+ RuntimeProfile profile (" test_profile" );
2131+ RuntimeState state {TQueryOptions (), TQueryGlobals ()};
2132+ auto scan_params = make_local_parquet_scan_params ();
2133+ io::FileReaderStats file_reader_stats;
2134+ io::FileCacheStatistics file_cache_stats;
2135+ auto io_ctx = make_io_context (&file_reader_stats, &file_cache_stats);
2136+ ShardedKVCache cache (1 );
2137+ doris::format::iceberg::IcebergTableReader reader;
2138+ ASSERT_TRUE (reader.init ({
2139+ .projected_columns = projected_columns,
2140+ .conjuncts = {},
2141+ .format = FileFormat::PARQUET ,
2142+ .scan_params = &scan_params,
2143+ .io_ctx = io_ctx,
2144+ .runtime_state = &state,
2145+ .scanner_profile = &profile,
2146+ })
2147+ .ok ());
2148+
2149+ auto split_options = build_split_options (file_path);
2150+ split_options.cache = &cache;
2151+ split_options.current_range .__set_table_format_params (make_iceberg_table_format_desc (
2152+ file_path, {make_iceberg_position_delete_file (delete_file_path)}));
2153+ auto status = reader.prepare_split (split_options);
2154+ ASSERT_FALSE (status.ok ());
2155+ EXPECT_NE (status.to_string ().find (" pos contains null values" ), std::string::npos)
2156+ << status.to_string ();
2157+
2158+ ASSERT_TRUE (reader.close ().ok ());
2159+ std::filesystem::remove_all (test_dir);
2160+ }
2161+
19832162TEST (IcebergV2ReaderTest, IcebergTableReaderIgnoresPositionDeleteFilesWhenDeletionVectorPresent) {
19842163 const auto test_dir =
19852164 std::filesystem::temp_directory_path () / " doris_iceberg_delete_files_merge_test" ;
0 commit comments