Skip to content

Commit 1072385

Browse files
committed
[bugfix](iceberg)Fixed random core with writing iceberg partitioned table (#39808)
Use local variables instead of member variables.
1 parent d18ded0 commit 1072385

2 files changed

Lines changed: 25 additions & 34 deletions

File tree

‎be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp‎

Lines changed: 22 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ Status VIcebergTableWriter::write(vectorized::Block& block) {
125125
auto writer_iter = _partitions_to_writers.find("");
126126
if (writer_iter == _partitions_to_writers.end()) {
127127
try {
128-
writer = _create_partition_writer(output_block, -1);
128+
writer = _create_partition_writer(nullptr, -1);
129129
} catch (doris::Exception& e) {
130130
return e.to_status();
131131
}
@@ -141,7 +141,7 @@ Status VIcebergTableWriter::write(vectorized::Block& block) {
141141
}
142142
_partitions_to_writers.erase(writer_iter);
143143
try {
144-
writer = _create_partition_writer(output_block, -1, &file_name,
144+
writer = _create_partition_writer(nullptr, -1, &file_name,
145145
file_name_index + 1);
146146
} catch (doris::Exception& e) {
147147
return e.to_status();
@@ -160,22 +160,21 @@ Status VIcebergTableWriter::write(vectorized::Block& block) {
160160
}
161161

162162
{
163+
Block transformed_block;
163164
SCOPED_RAW_TIMER(&_partition_writers_dispatch_ns);
164-
_transformed_block.clear();
165-
_transformed_block.reserve(_iceberg_partition_columns.size());
165+
transformed_block.reserve(_iceberg_partition_columns.size());
166166
for (auto& iceberg_partition_columns : _iceberg_partition_columns) {
167-
_transformed_block.insert(iceberg_partition_columns.partition_column_transform().apply(
167+
transformed_block.insert(iceberg_partition_columns.partition_column_transform().apply(
168168
output_block, iceberg_partition_columns.source_idx()));
169169
}
170170
for (int i = 0; i < output_block.rows(); ++i) {
171171
std::optional<PartitionData> partition_data;
172172
try {
173-
partition_data = _get_partition_data(_transformed_block, i);
173+
partition_data = _get_partition_data(&transformed_block, i);
174174
} catch (doris::Exception& e) {
175175
return e.to_status();
176176
}
177177
std::string partition_name;
178-
DCHECK(partition_data.has_value());
179178
try {
180179
partition_name = _partition_to_path(partition_data.value());
181180
} catch (doris::Exception& e) {
@@ -186,7 +185,7 @@ Status VIcebergTableWriter::write(vectorized::Block& block) {
186185
const std::string* file_name, int file_name_index,
187186
std::shared_ptr<VIcebergPartitionWriter>& writer_ptr) -> Status {
188187
try {
189-
auto writer = _create_partition_writer(output_block, position, file_name,
188+
auto writer = _create_partition_writer(&transformed_block, position, file_name,
190189
file_name_index);
191190
RETURN_IF_ERROR(writer->open(_state, _profile));
192191
IColumn::Filter filter(output_block.rows(), 0);
@@ -342,30 +341,27 @@ std::vector<std::string> VIcebergTableWriter::_partition_values(
342341
}
343342

344343
std::shared_ptr<VIcebergPartitionWriter> VIcebergTableWriter::_create_partition_writer(
345-
vectorized::Block& block, int position, const std::string* file_name, int file_name_index) {
344+
vectorized::Block* transformed_block, int position, const std::string* file_name,
345+
int file_name_index) {
346346
auto& iceberg_table_sink = _t_sink.iceberg_table_sink;
347-
std::optional<PartitionData> partition_data;
348-
partition_data = _get_partition_data(_transformed_block, position);
349-
std::string partition_path;
350347
std::vector<std::string> partition_values;
351-
if (partition_data.has_value()) {
352-
partition_path = _partition_to_path(partition_data.value());
353-
partition_values = _partition_values(partition_data.value());
354-
}
355348
const std::string& output_path = iceberg_table_sink.output_path;
356-
357349
std::string write_path;
358350
std::string original_write_path;
359351
std::string target_path;
360-
if (partition_path.empty()) {
361-
original_write_path = iceberg_table_sink.original_output_path;
362-
target_path = output_path;
363-
write_path = output_path;
364-
} else {
352+
353+
if (transformed_block != nullptr) {
354+
PartitionData partition_data = _get_partition_data(transformed_block, position);
355+
std::string partition_path = _partition_to_path(partition_data);
356+
partition_values = _partition_values(partition_data);
365357
original_write_path =
366358
fmt::format("{}/{}", iceberg_table_sink.original_output_path, partition_path);
367359
target_path = fmt::format("{}/{}", output_path, partition_path);
368360
write_path = fmt::format("{}/{}", output_path, partition_path);
361+
} else {
362+
original_write_path = iceberg_table_sink.original_output_path;
363+
target_path = output_path;
364+
write_path = output_path;
369365
}
370366

371367
VIcebergPartitionWriter::WriteInfo write_info = {
@@ -388,18 +384,15 @@ std::shared_ptr<VIcebergPartitionWriter> VIcebergTableWriter::_create_partition_
388384
iceberg_table_sink.hadoop_config);
389385
}
390386

391-
std::optional<PartitionData> VIcebergTableWriter::_get_partition_data(
392-
vectorized::Block& transformed_block, int position) {
393-
if (_iceberg_partition_columns.empty()) {
394-
return std::nullopt;
395-
}
396-
387+
PartitionData VIcebergTableWriter::_get_partition_data(vectorized::Block* transformed_block,
388+
int position) {
389+
DCHECK(!_iceberg_partition_columns.empty());
397390
std::vector<std::any> values;
398391
values.reserve(_iceberg_partition_columns.size());
399392
int column_idx = 0;
400393
for (auto& iceberg_partition_column : _iceberg_partition_columns) {
401394
const vectorized::ColumnWithTypeAndName& partition_column =
402-
transformed_block.get_by_position(column_idx);
395+
transformed_block->get_by_position(column_idx);
403396
const TypeDescriptor& result_type =
404397
iceberg_partition_column.partition_column_transform().get_result_type();
405398
auto value = _get_iceberg_partition_value(result_type, partition_column, position);

‎be/src/vec/sink/writer/iceberg/viceberg_table_writer.h‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -97,10 +97,10 @@ class VIcebergTableWriter final : public AsyncResultWriter {
9797
std::vector<std::string> _partition_values(const doris::iceberg::StructLike& data);
9898

9999
std::shared_ptr<VIcebergPartitionWriter> _create_partition_writer(
100-
vectorized::Block& block, int position, const std::string* file_name = nullptr,
101-
int file_name_index = 0);
100+
vectorized::Block* transformed_block, int position,
101+
const std::string* file_name = nullptr, int file_name_index = 0);
102102

103-
std::optional<PartitionData> _get_partition_data(vectorized::Block& block, int position);
103+
PartitionData _get_partition_data(vectorized::Block* block, int position);
104104

105105
std::any _get_iceberg_partition_value(const TypeDescriptor& type_desc,
106106
const ColumnWithTypeAndName& partition_column,
@@ -127,8 +127,6 @@ class VIcebergTableWriter final : public AsyncResultWriter {
127127

128128
VExprContextSPtrs _write_output_vexpr_ctxs;
129129

130-
Block _transformed_block;
131-
132130
size_t _row_count = 0;
133131

134132
// profile counters

0 commit comments

Comments
 (0)