Skip to content

Commit 87ea629

Browse files
committed
[test](iceberg) Cover nullable position delete reader path
1 parent 57248ec commit 87ea629

1 file changed

Lines changed: 179 additions & 0 deletions

File tree

‎be/test/format_v2/table/iceberg_reader_test.cpp‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,19 @@ std::shared_ptr<arrow::Array> build_nullable_int64_array(
260260
return finish_array(&builder);
261261
}
262262

263+
std::shared_ptr<arrow::Array> build_nullable_string_array(
264+
const std::vector<std::optional<std::string>>& values) {
265+
arrow::StringBuilder builder;
266+
for (const auto& value : values) {
267+
if (value.has_value()) {
268+
EXPECT_TRUE(builder.Append(*value).ok());
269+
} else {
270+
EXPECT_TRUE(builder.AppendNull().ok());
271+
}
272+
}
273+
return finish_array(&builder);
274+
}
275+
263276
std::shared_ptr<arrow::Array> build_string_array(const std::vector<std::string>& values) {
264277
arrow::StringBuilder builder;
265278
for (const auto& value : values) {
@@ -405,6 +418,31 @@ void write_position_delete_parquet_file(const std::string& file_path,
405418
builder.build()));
406419
}
407420

421+
void write_nullable_position_delete_parquet_file(
422+
const std::string& file_path,
423+
const std::vector<std::optional<std::string>>& data_file_paths,
424+
const std::vector<std::optional<int64_t>>& positions) {
425+
ASSERT_EQ(data_file_paths.size(), positions.size());
426+
auto schema = arrow::schema({
427+
arrow::field("file_path", arrow::utf8(), true),
428+
arrow::field("pos", arrow::int64(), true),
429+
});
430+
auto table = arrow::Table::Make(schema, {build_nullable_string_array(data_file_paths),
431+
build_nullable_int64_array(positions)});
432+
433+
auto file_result = arrow::io::FileOutputStream::Open(file_path);
434+
ASSERT_TRUE(file_result.ok()) << file_result.status();
435+
std::shared_ptr<arrow::io::FileOutputStream> out = *file_result;
436+
437+
::parquet::WriterProperties::Builder builder;
438+
builder.version(::parquet::ParquetVersion::PARQUET_2_6);
439+
builder.data_page_version(::parquet::ParquetDataPageVersion::V2);
440+
builder.compression(::parquet::Compression::UNCOMPRESSED);
441+
PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out,
442+
static_cast<int64_t>(positions.size()),
443+
builder.build()));
444+
}
445+
408446
int64_t write_iceberg_deletion_vector_file(const std::string& file_path,
409447
const std::vector<uint64_t>& deleted_positions) {
410448
roaring::Roaring64Map rows;
@@ -1980,6 +2018,147 @@ TEST(IcebergV2ReaderTest, IcebergTableReaderAppliesPositionDeleteFile) {
19802018
std::filesystem::remove_all(test_dir);
19812019
}
19822020

2021+
TEST(IcebergV2ReaderTest, IcebergTableReaderAppliesNullablePositionDeleteFileWithoutNulls) {
2022+
const auto test_dir = std::filesystem::temp_directory_path() /
2023+
"doris_iceberg_nullable_position_delete_file_test";
2024+
std::filesystem::remove_all(test_dir);
2025+
std::filesystem::create_directories(test_dir);
2026+
2027+
const auto file_path = (test_dir / "split.parquet").string();
2028+
const auto delete_file_path = (test_dir / "position-delete.parquet").string();
2029+
write_int_pair_parquet_file(file_path, {1, 2, 3, 4, 5}, {10, 20, 30, 40, 50},
2030+
{"one", "two", "three", "four", "five"});
2031+
write_nullable_position_delete_parquet_file(delete_file_path, {file_path, file_path},
2032+
{int64_t {1}, int64_t {3}});
2033+
2034+
std::vector<ColumnDefinition> projected_columns;
2035+
projected_columns.push_back(make_table_column(0, "id", std::make_shared<DataTypeInt32>()));
2036+
2037+
RuntimeProfile profile("test_profile");
2038+
RuntimeState state {TQueryOptions(), TQueryGlobals()};
2039+
auto scan_params = make_local_parquet_scan_params();
2040+
io::FileReaderStats file_reader_stats;
2041+
io::FileCacheStatistics file_cache_stats;
2042+
auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats);
2043+
ShardedKVCache cache(1);
2044+
doris::format::iceberg::IcebergTableReader reader;
2045+
ASSERT_TRUE(reader.init({
2046+
.projected_columns = projected_columns,
2047+
.conjuncts = {},
2048+
.format = FileFormat::PARQUET,
2049+
.scan_params = &scan_params,
2050+
.io_ctx = io_ctx,
2051+
.runtime_state = &state,
2052+
.scanner_profile = &profile,
2053+
})
2054+
.ok());
2055+
2056+
auto split_options = build_split_options(file_path);
2057+
split_options.cache = &cache;
2058+
split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc(
2059+
file_path, {make_iceberg_position_delete_file(delete_file_path)}));
2060+
ASSERT_TRUE(reader.prepare_split(split_options).ok());
2061+
2062+
EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector<int32_t>({1, 3, 5}));
2063+
2064+
ASSERT_TRUE(reader.close().ok());
2065+
std::filesystem::remove_all(test_dir);
2066+
}
2067+
2068+
TEST(IcebergV2ReaderTest, IcebergTableReaderRejectsNullablePositionDeleteFileWithActualNulls) {
2069+
const auto test_dir = std::filesystem::temp_directory_path() /
2070+
"doris_iceberg_nullable_position_delete_actual_null_test";
2071+
std::filesystem::remove_all(test_dir);
2072+
std::filesystem::create_directories(test_dir);
2073+
2074+
const auto file_path = (test_dir / "split.parquet").string();
2075+
const auto delete_file_path = (test_dir / "position-delete.parquet").string();
2076+
write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"});
2077+
write_nullable_position_delete_parquet_file(delete_file_path, {file_path, std::nullopt},
2078+
{int64_t {1}, int64_t {2}});
2079+
2080+
std::vector<ColumnDefinition> projected_columns;
2081+
projected_columns.push_back(make_table_column(0, "id", std::make_shared<DataTypeInt32>()));
2082+
2083+
RuntimeProfile profile("test_profile");
2084+
RuntimeState state {TQueryOptions(), TQueryGlobals()};
2085+
auto scan_params = make_local_parquet_scan_params();
2086+
io::FileReaderStats file_reader_stats;
2087+
io::FileCacheStatistics file_cache_stats;
2088+
auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats);
2089+
ShardedKVCache cache(1);
2090+
doris::format::iceberg::IcebergTableReader reader;
2091+
ASSERT_TRUE(reader.init({
2092+
.projected_columns = projected_columns,
2093+
.conjuncts = {},
2094+
.format = FileFormat::PARQUET,
2095+
.scan_params = &scan_params,
2096+
.io_ctx = io_ctx,
2097+
.runtime_state = &state,
2098+
.scanner_profile = &profile,
2099+
})
2100+
.ok());
2101+
2102+
auto split_options = build_split_options(file_path);
2103+
split_options.cache = &cache;
2104+
split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc(
2105+
file_path, {make_iceberg_position_delete_file(delete_file_path)}));
2106+
auto status = reader.prepare_split(split_options);
2107+
ASSERT_FALSE(status.ok());
2108+
EXPECT_NE(status.to_string().find("file_path contains null values"), std::string::npos)
2109+
<< status.to_string();
2110+
2111+
ASSERT_TRUE(reader.close().ok());
2112+
std::filesystem::remove_all(test_dir);
2113+
}
2114+
2115+
TEST(IcebergV2ReaderTest, IcebergTableReaderRejectsNullablePositionDeletePosWithActualNulls) {
2116+
const auto test_dir = std::filesystem::temp_directory_path() /
2117+
"doris_iceberg_nullable_position_delete_pos_null_test";
2118+
std::filesystem::remove_all(test_dir);
2119+
std::filesystem::create_directories(test_dir);
2120+
2121+
const auto file_path = (test_dir / "split.parquet").string();
2122+
const auto delete_file_path = (test_dir / "position-delete.parquet").string();
2123+
write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"});
2124+
write_nullable_position_delete_parquet_file(delete_file_path, {file_path, file_path},
2125+
{int64_t {1}, std::nullopt});
2126+
2127+
std::vector<ColumnDefinition> projected_columns;
2128+
projected_columns.push_back(make_table_column(0, "id", std::make_shared<DataTypeInt32>()));
2129+
2130+
RuntimeProfile profile("test_profile");
2131+
RuntimeState state {TQueryOptions(), TQueryGlobals()};
2132+
auto scan_params = make_local_parquet_scan_params();
2133+
io::FileReaderStats file_reader_stats;
2134+
io::FileCacheStatistics file_cache_stats;
2135+
auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats);
2136+
ShardedKVCache cache(1);
2137+
doris::format::iceberg::IcebergTableReader reader;
2138+
ASSERT_TRUE(reader.init({
2139+
.projected_columns = projected_columns,
2140+
.conjuncts = {},
2141+
.format = FileFormat::PARQUET,
2142+
.scan_params = &scan_params,
2143+
.io_ctx = io_ctx,
2144+
.runtime_state = &state,
2145+
.scanner_profile = &profile,
2146+
})
2147+
.ok());
2148+
2149+
auto split_options = build_split_options(file_path);
2150+
split_options.cache = &cache;
2151+
split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc(
2152+
file_path, {make_iceberg_position_delete_file(delete_file_path)}));
2153+
auto status = reader.prepare_split(split_options);
2154+
ASSERT_FALSE(status.ok());
2155+
EXPECT_NE(status.to_string().find("pos contains null values"), std::string::npos)
2156+
<< status.to_string();
2157+
2158+
ASSERT_TRUE(reader.close().ok());
2159+
std::filesystem::remove_all(test_dir);
2160+
}
2161+
19832162
TEST(IcebergV2ReaderTest, IcebergTableReaderIgnoresPositionDeleteFilesWhenDeletionVectorPresent) {
19842163
const auto test_dir =
19852164
std::filesystem::temp_directory_path() / "doris_iceberg_delete_files_merge_test";

0 commit comments

Comments
 (0)