Skip to content

Commit 897bb25

Browse files
authored
[fix](iceberg) Validate Parquet compression before file creation (apache#69376)
### What problem does this PR solve? Issue Number: N/A Related PR: apache#68854, apache#69338 Problem Summary: The Iceberg data, position-delete, and Hive Parquet writers created their output file before validating the requested Parquet compression codec. For an unsupported codec, the Iceberg data writer then attempted to delete an open file, while the other writers returned after creating the file. This made error cleanup depend on filesystem behavior and could leave an orphan file. Resolve the codec before calling `create_file` so invalid codecs fail without creating any output file, and add a regression test for the failure path. ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [x] Unit Test: added `UnsupportedParquetCodecDoesNotCreateFile`; CI will run it. - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [x] No. - [ ] Yes. - Does this need documentation? - [x] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label
1 parent dac77e0 commit 897bb25

4 files changed

Lines changed: 59 additions & 12 deletions

File tree

‎be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,13 +80,14 @@ Status VIcebergDeleteFileWriter::open(RuntimeState* state, RuntimeProfile* profi
8080
_fs = DORIS_TRY(FileFactory::create_fs(fs_properties, file_description));
8181

8282
io::FileWriterOptions file_writer_options = {.used_by_s3_committer = false};
83-
RETURN_IF_ERROR(_fs->create_file(file_description.path, &_file_writer, &file_writer_options));
8483

8584
// Create file format transformer based on format type
8685
switch (_file_format) {
8786
case TFileFormatType::FORMAT_PARQUET: {
8887
TParquetCompressionType::type parquet_compression_type =
8988
DORIS_TRY(ParquetBuildHelper::to_parquet_compression_type(_compress_type));
89+
RETURN_IF_ERROR(
90+
_fs->create_file(file_description.path, &_file_writer, &file_writer_options));
9091

9192
ParquetFileOptions parquet_options = {parquet_compression_type,
9293
TParquetVersion::PARQUET_1_0, false, false};
@@ -96,6 +97,8 @@ Status VIcebergDeleteFileWriter::open(RuntimeState* state, RuntimeProfile* profi
9697
return _file_format_transformer->open();
9798
}
9899
case TFileFormatType::FORMAT_ORC: {
100+
RETURN_IF_ERROR(
101+
_fs->create_file(file_description.path, &_file_writer, &file_writer_options));
99102
_file_format_transformer.reset(new VOrcTransformer(state, _file_writer.get(), output_exprs,
100103
"", column_names, false, _compress_type,
101104
_position_delete_schema.get()));

‎be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -97,22 +97,17 @@ Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profil
9797
io::FileDescription file_description = {.path = _path, .fs_name {}};
9898
_fs = DORIS_TRY(FileFactory::create_fs(fs_properties, file_description));
9999
io::FileWriterOptions file_writer_options = {.used_by_s3_committer = false};
100-
RETURN_IF_ERROR(_fs->create_file(file_description.path, &_file_writer, &file_writer_options));
101100

102101
Status open_status;
103102
switch (_file_format_type) {
104103
case TFileFormatType::FORMAT_PARQUET: {
105-
auto parquet_compression_type =
106-
ParquetBuildHelper::to_parquet_compression_type(_compress_type);
107-
if (!parquet_compression_type.has_value()) {
108-
// The target file is already created, so let the shared cleanup below remove it
109-
// instead of returning early.
110-
open_status = parquet_compression_type.error();
111-
break;
112-
}
104+
TParquetCompressionType::type parquet_compression_type =
105+
DORIS_TRY(ParquetBuildHelper::to_parquet_compression_type(_compress_type));
106+
RETURN_IF_ERROR(
107+
_fs->create_file(file_description.path, &_file_writer, &file_writer_options));
113108
// Parquet 1.0 coerces Arrow nanoseconds to microseconds, including nested fields.
114109
ParquetFileOptions parquet_options = {
115-
.compression_type = parquet_compression_type.value(),
110+
.compression_type = parquet_compression_type,
116111
.parquet_version = contains_nanosecond_timestamp(_schema.root_struct())
117112
? TParquetVersion::PARQUET_2_LATEST
118113
: TParquetVersion::PARQUET_1_0,
@@ -125,6 +120,8 @@ Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profil
125120
break;
126121
}
127122
case TFileFormatType::FORMAT_ORC: {
123+
RETURN_IF_ERROR(
124+
_fs->create_file(file_description.path, &_file_writer, &file_writer_options));
128125
_file_format_transformer = std::make_unique<VOrcTransformer>(
129126
state, _file_writer.get(), _write_output_expr_ctxs, "", _write_column_names, false,
130127
_compress_type, &_schema, _fs, _nan_count_field_ids);

‎be/src/exec/sink/writer/vhive_partition_writer.cpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,12 +85,13 @@ Status VHivePartitionWriter::open(RuntimeState* state, RuntimeProfile* operator_
8585
"completion");
8686
}
8787
io::FileWriterOptions file_writer_options = {.used_by_s3_committer = true};
88-
RETURN_IF_ERROR(_fs->create_file(file_description.path, &_file_writer, &file_writer_options));
8988

9089
switch (_file_format_type) {
9190
case TFileFormatType::FORMAT_PARQUET: {
9291
TParquetCompressionType::type parquet_compression_type =
9392
DORIS_TRY(ParquetBuildHelper::to_parquet_compression_type(_hive_compress_type));
93+
RETURN_IF_ERROR(
94+
_fs->create_file(file_description.path, &_file_writer, &file_writer_options));
9495
// TODO: INT96 is kept for Hive 2/3 compatibility. Add an explicit option before
9596
// changing the default Hive parquet timestamp encoding to standard logical types.
9697
ParquetFileOptions parquet_options = {parquet_compression_type,
@@ -107,12 +108,16 @@ Status VHivePartitionWriter::open(RuntimeState* state, RuntimeProfile* operator_
107108
return _file_format_transformer->open();
108109
}
109110
case TFileFormatType::FORMAT_ORC: {
111+
RETURN_IF_ERROR(
112+
_fs->create_file(file_description.path, &_file_writer, &file_writer_options));
110113
_file_format_transformer = std::make_unique<VOrcTransformer>(
111114
state, _file_writer.get(), _write_output_expr_ctxs, "", _write_column_names, false,
112115
_hive_compress_type);
113116
return _file_format_transformer->open();
114117
}
115118
case TFileFormatType::FORMAT_CSV_PLAIN: {
119+
RETURN_IF_ERROR(
120+
_fs->create_file(file_description.path, &_file_writer, &file_writer_options));
116121
// TODO(syt): support hive csv table, only hive text file is supportted now
117122
_file_format_transformer = std::make_unique<VCSVTransformer>(
118123
state, _file_writer.get(), _write_output_expr_ctxs, false, "csv", "",

‎be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include <optional>
2222

2323
#include "core/data_type/data_type_number.h"
24+
#include "cpp/sync_point.h"
2425
#include "exec/sink/writer/iceberg/viceberg_partition_writer.h"
2526
#include "exec/sink/writer/iceberg/viceberg_sort_writer.h"
2627
#include "format/table/iceberg/schema_parser.h"
@@ -205,6 +206,47 @@ TEST_F(VIcebergPartitionWriterTest, MissingPolicyKeepsCollectionEnabledForRollin
205206
EXPECT_TRUE(collect_column_stats(*writer));
206207
}
207208

209+
TEST_F(VIcebergPartitionWriterTest, UnsupportedParquetCodecDoesNotCreateFile) {
210+
VExprContextSPtrs output_exprs;
211+
iceberg::Schema schema(std::vector<iceberg::NestedField> {});
212+
std::string schema_json;
213+
std::map<std::string, std::string> hadoop_conf;
214+
IPartitionWriterBase::WriteInfo write_info {.location = IcebergOutputLocation(".", "."),
215+
.file_type = TFileType::FILE_LOCAL,
216+
.broker_addresses = {}};
217+
auto writer = std::make_unique<VIcebergPartitionWriter>(
218+
make_table_sink(std::nullopt), std::vector<std::string> {}, output_exprs, schema,
219+
&schema_json, std::vector<std::string> {}, std::move(write_info),
220+
"unsupported_parquet_codec_" + UniqueId::gen_uid().to_string(), 0,
221+
TFileFormatType::FORMAT_PARQUET, TFileCompressType::ZLIB, hadoop_conf);
222+
223+
auto* sync_point = SyncPoint::get_instance();
224+
const bool sync_point_was_enabled = sync_point->get_enable();
225+
int create_file_calls = 0;
226+
SyncPoint::CallbackGuard callback_guard;
227+
sync_point->set_call_back(
228+
"LocalFileSystem::create_file_impl", [&](auto&&) { ++create_file_calls; },
229+
&callback_guard);
230+
sync_point->enable_processing();
231+
Defer restore_sync_point([&] {
232+
if (!sync_point_was_enabled) {
233+
sync_point->disable_processing();
234+
}
235+
});
236+
237+
MockRuntimeState state;
238+
Status status = writer->open(&state, nullptr, nullptr);
239+
240+
EXPECT_EQ(0, create_file_calls);
241+
EXPECT_FALSE(status.ok());
242+
EXPECT_NE(std::string::npos, status.to_string().find("Unsupported compress type"));
243+
EXPECT_EQ(nullptr, writer->_file_writer);
244+
ASSERT_NE(nullptr, writer->_fs);
245+
bool exists = true;
246+
ASSERT_TRUE(writer->_fs->exists(writer->_path, &exists).ok());
247+
EXPECT_FALSE(exists);
248+
}
249+
208250
TEST_F(VIcebergPartitionWriterTest, SortWriterPropagatesUnderlyingCloseFailure) {
209251
VExprContextSPtrs output_exprs;
210252
iceberg::Schema schema(std::vector<iceberg::NestedField> {});

0 commit comments

Comments
 (0)