Skip to content

Commit 645a4c4

Browse files
authored
[Fix](multi-catalog)Fix the issue of not initializing the writer caused by refactoring and add hive writing regression test. (#32721)
Issue Number: #31442 - Fix the issue of not initializing the writer caused by refactoring code in #31716. - Fix reference lifetime issue of `TParquetVersion::type parquet_version` in `VParquetTransformer ` when using temp object. - Add hive writing regression tests.
1 parent 91be50b commit 645a4c4

18 files changed

Lines changed: 1981 additions & 27 deletions

File tree

‎be/src/vec/runtime/vparquet_transformer.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -200,9 +200,9 @@ void ParquetBuildHelper::build_version(parquet::WriterProperties::Builder& build
200200
VParquetTransformer::VParquetTransformer(RuntimeState* state, doris::io::FileWriter* file_writer,
201201
const VExprContextSPtrs& output_vexpr_ctxs,
202202
const std::vector<TParquetSchema>& parquet_schemas,
203-
const TParquetCompressionType::type& compression_type,
204-
const bool& parquet_disable_dictionary,
205-
const TParquetVersion::type& parquet_version,
203+
TParquetCompressionType::type compression_type,
204+
bool parquet_disable_dictionary,
205+
TParquetVersion::type parquet_version,
206206
bool output_object_data)
207207
: VFileFormatTransformer(state, output_vexpr_ctxs, output_object_data),
208208
_parquet_schemas(parquet_schemas),

‎be/src/vec/runtime/vparquet_transformer.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -92,9 +92,9 @@ class VParquetTransformer final : public VFileFormatTransformer {
9292
VParquetTransformer(RuntimeState* state, doris::io::FileWriter* file_writer,
9393
const VExprContextSPtrs& output_vexpr_ctxs,
9494
const std::vector<TParquetSchema>& parquet_schemas,
95-
const TParquetCompressionType::type& compression_type,
96-
const bool& parquet_disable_dictionary,
97-
const TParquetVersion::type& parquet_version, bool output_object_data);
95+
TParquetCompressionType::type compression_type,
96+
bool parquet_disable_dictionary, TParquetVersion::type parquet_version,
97+
bool output_object_data);
9898

9999
~VParquetTransformer() override = default;
100100

@@ -118,9 +118,9 @@ class VParquetTransformer final : public VFileFormatTransformer {
118118
std::shared_ptr<arrow::Schema> _arrow_schema;
119119

120120
const std::vector<TParquetSchema>& _parquet_schemas;
121-
const TParquetCompressionType::type& _compression_type;
122-
const bool& _parquet_disable_dictionary;
123-
const TParquetVersion::type& _parquet_version;
121+
const TParquetCompressionType::type _compression_type;
122+
const bool _parquet_disable_dictionary;
123+
const TParquetVersion::type _parquet_version;
124124
};
125125

126126
} // namespace doris::vectorized

‎be/src/vec/sink/writer/vhive_partition_writer.cpp‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,10 @@ Status VHivePartitionWriter::open(RuntimeState* state, RuntimeProfile* profile)
4949

5050
io::FSPropertiesRef fs_properties(_write_info.file_type);
5151
fs_properties.properties = &_hadoop_conf;
52-
_fs = DORIS_TRY(FileFactory::create_fs(fs_properties,
53-
{.path = _write_info.write_path + '/' + _file_name}));
52+
io::FileDescription file_description = {
53+
.path = fmt::format("{}/{}", _write_info.write_path, _file_name)};
54+
_fs = DORIS_TRY(FileFactory::create_fs(fs_properties, file_description));
55+
RETURN_IF_ERROR(_fs->create_file(file_description.path, &_file_writer));
5456

5557
switch (_file_format_type) {
5658
case TFileFormatType::FORMAT_PARQUET: {
@@ -75,17 +77,18 @@ Status VHivePartitionWriter::open(RuntimeState* state, RuntimeProfile* profile)
7577
}
7678
}
7779
std::vector<TParquetSchema> parquet_schemas;
80+
parquet_schemas.reserve(_columns.size());
7881
for (int i = 0; i < _columns.size(); i++) {
7982
VExprSPtr column_expr = _vec_output_expr_ctxs[i]->root();
8083
TParquetSchema parquet_schema;
8184
parquet_schema.schema_column_name = _columns[i].name;
8285
parquet_schemas.emplace_back(std::move(parquet_schema));
8386
}
84-
_vfile_writer.reset(new VParquetTransformer(
85-
state, _file_writer_impl.get(), _vec_output_expr_ctxs, parquet_schemas,
87+
_file_format_transformer.reset(new VParquetTransformer(
88+
state, _file_writer.get(), _vec_output_expr_ctxs, parquet_schemas,
8689
parquet_compression_type, parquet_disable_dictionary, TParquetVersion::PARQUET_1_0,
8790
false));
88-
return _vfile_writer->open();
91+
return _file_format_transformer->open();
8992
}
9093
case TFileFormatType::FORMAT_ORC: {
9194
orc::CompressionKind orc_compression_type;
@@ -122,10 +125,10 @@ Status VHivePartitionWriter::open(RuntimeState* state, RuntimeProfile* profile)
122125
}
123126
}
124127

125-
_vfile_writer.reset(new VOrcTransformer(state, _file_writer_impl.get(),
126-
_vec_output_expr_ctxs, std::move(root_schema),
127-
false, orc_compression_type));
128-
return _vfile_writer->open();
128+
_file_format_transformer.reset(
129+
new VOrcTransformer(state, _file_writer.get(), _vec_output_expr_ctxs,
130+
std::move(root_schema), false, orc_compression_type));
131+
return _file_format_transformer->open();
129132
}
130133
default: {
131134
return Status::InternalError("Unsupported file format type {}",
@@ -135,13 +138,14 @@ Status VHivePartitionWriter::open(RuntimeState* state, RuntimeProfile* profile)
135138
}
136139

137140
Status VHivePartitionWriter::close(const Status& status) {
138-
if (_vfile_writer != nullptr) {
139-
Status st = _vfile_writer->close();
141+
if (_file_format_transformer != nullptr) {
142+
Status st = _file_format_transformer->close();
140143
if (!st.ok()) {
141-
LOG(WARNING) << fmt::format("_vfile_writer close failed, reason: {}", st.to_string());
144+
LOG(WARNING) << fmt::format("_file_format_transformer close failed, reason: {}",
145+
st.to_string());
142146
}
143147
}
144-
if (!status.ok()) {
148+
if (!status.ok() && _fs != nullptr) {
145149
auto path = fmt::format("{}/{}", _write_info.write_path, _file_name);
146150
Status st = _fs->delete_file(path);
147151
if (!st.ok()) {
@@ -155,7 +159,7 @@ Status VHivePartitionWriter::close(const Status& status) {
155159
Status VHivePartitionWriter::write(vectorized::Block& block, vectorized::IColumn::Filter* filter) {
156160
Block output_block;
157161
RETURN_IF_ERROR(_projection_and_filter_block(block, filter, &output_block));
158-
RETURN_IF_ERROR(_vfile_writer->write(output_block));
162+
RETURN_IF_ERROR(_file_format_transformer->write(output_block));
159163
_row_count += output_block.rows();
160164
_input_size_in_bytes += output_block.bytes();
161165
return Status::OK();

‎be/src/vec/sink/writer/vhive_partition_writer.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ class VHivePartitionWriter {
6262

6363
Status close(const Status& status);
6464

65-
inline size_t written_len() { return _vfile_writer->written_len(); }
65+
inline size_t written_len() { return _file_format_transformer->written_len(); }
6666

6767
private:
6868
std::unique_ptr<orc::Type> _build_orc_type(const TypeDescriptor& type_descriptor);
@@ -91,13 +91,13 @@ class VHivePartitionWriter {
9191
TFileCompressType::type _hive_compress_type;
9292
const std::map<std::string, std::string>& _hadoop_conf;
9393

94-
std::shared_ptr<io::FileSystem> _fs;
94+
std::shared_ptr<io::FileSystem> _fs = nullptr;
9595

9696
// If the result file format is plain text, like CSV, this _file_writer is owned by this FileResultWriter.
9797
// If the result file format is Parquet, this _file_writer is owned by _parquet_writer.
98-
std::unique_ptr<io::FileWriter> _file_writer_impl = nullptr;
98+
std::unique_ptr<doris::io::FileWriter> _file_writer = nullptr;
9999
// convert block to parquet/orc/csv format
100-
std::unique_ptr<VFileFormatTransformer> _vfile_writer = nullptr;
100+
std::unique_ptr<VFileFormatTransformer> _file_format_transformer = nullptr;
101101

102102
RuntimeState* _state;
103103
};

0 commit comments

Comments
 (0)