Skip to content

Commit 6a32fb5

Browse files
kaka11chenHappenLee
authored andcommitted
[Fix](hive-writer) Fix hive partition update core. (apache#35311)
Issue: apache#31442 ``` /home/zcp/repo_center/doris_master/doris/be/src/common/signal_handler.h:421 1# PosixSignals::chained_handler(int, siginfo*, void*) [clone .part.0] in /usr/lib/jvm/java-17-openjdk-amd64/lib/server/libjvm.so 2# JVM_handle_linux_signal in /usr/lib/jvm/java-17-openjdk-amd64/lib/server/libjvm.so 3# 0x00007F963FA9D090 in /lib/x86_64-linux-gnu/libc.so.6 4# doris::vectorized::VHivePartitionWriter::_build_partition_update() at /home/zcp/repo_center/doris_master/doris/be/src/vec/sink/writer/vhive_partition_writer.cpp:215 5# doris::vectorized::VHivePartitionWriter::close(doris::Status const&) at /home/zcp/repo_center/doris_master/doris/be/src/vec/sink/writer/vhive_partition_writer.cpp:164 6# doris::vectorized::VHiveTableWriter::close(doris::Status) at /home/zcp/repo_center/doris_master/doris/be/src/vec/sink/writer/vhive_table_writer.cpp:209 7# doris::vectorized::AsyncResultWriter::process_block(doris::RuntimeState*, doris::RuntimeProfile*) at /home/zcp/repo_center/doris_master/doris/be/src/vec/sink/writer/async_result_writer.cpp:184 8# doris::vectorized::AsyncResultWriter::start_writer(doris::RuntimeState*, doris::RuntimeProfile*)::$_0::operator()() const at ```
1 parent 61105f4 commit 6a32fb5

2 files changed

Lines changed: 11 additions & 6 deletions

File tree

‎be/src/vec/sink/writer/vhive_partition_writer.cpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,9 @@ Status VHivePartitionWriter::open(RuntimeState* state, RuntimeProfile* profile)
147147
}
148148

149149
Status VHivePartitionWriter::close(const Status& status) {
150+
if (status.ok()) {
151+
_state->hive_partition_updates().emplace_back(_build_partition_update());
152+
}
150153
if (_file_format_transformer != nullptr) {
151154
Status st = _file_format_transformer->close();
152155
if (!st.ok()) {
@@ -161,7 +164,6 @@ Status VHivePartitionWriter::close(const Status& status) {
161164
LOG(WARNING) << fmt::format("Delete file {} failed, reason: {}", path, st.to_string());
162165
}
163166
}
164-
_state->hive_partition_updates().emplace_back(_build_partition_update());
165167
return Status::OK();
166168
}
167169

@@ -212,11 +214,14 @@ THivePartitionUpdate VHivePartitionWriter::_build_partition_update() {
212214
hive_partition_update.__set_location(location);
213215
hive_partition_update.__set_file_names({_get_target_file_name()});
214216
hive_partition_update.__set_row_count(_row_count);
217+
DCHECK(_file_format_transformer != nullptr);
215218
hive_partition_update.__set_file_size(_file_format_transformer->written_len());
216219

217220
if (_write_info.file_type == TFileType::FILE_S3) {
221+
DCHECK(_file_writer != nullptr);
218222
doris::io::S3FileWriter* s3_mpu_file_writer =
219223
dynamic_cast<doris::io::S3FileWriter*>(_file_writer.get());
224+
DCHECK(s3_mpu_file_writer != nullptr);
220225
TS3MPUPendingUpload s3_mpu_pending_upload;
221226
s3_mpu_pending_upload.__set_bucket(s3_mpu_file_writer->bucket());
222227
s3_mpu_pending_upload.__set_key(s3_mpu_file_writer->key());

‎be/src/vec/sink/writer/vhive_table_writer.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,8 @@ Status VHiveTableWriter::close(Status status) {
208208
for (const auto& pair : _partitions_to_writers) {
209209
Status st = pair.second->close(status);
210210
if (st != Status::OK()) {
211-
LOG(WARNING) << fmt::format("Unsupported type for partition {}", st.to_string());
211+
LOG(WARNING) << fmt::format("partition writer close failed for partition {}",
212+
st.to_string());
212213
continue;
213214
}
214215
}
@@ -330,15 +331,15 @@ std::vector<std::string> VHiveTableWriter::_create_partition_values(vectorized::
330331
partition_column, position);
331332

332333
// Check if value contains only printable ASCII characters
333-
bool isValid = true;
334+
bool is_valid = true;
334335
for (char c : value) {
335336
if (c < 0x20 || c > 0x7E) {
336-
isValid = false;
337+
is_valid = false;
337338
break;
338339
}
339340
}
340341

341-
if (!isValid) {
342+
if (!is_valid) {
342343
// Encode value using Base16 encoding with space separator
343344
std::stringstream encoded;
344345
for (unsigned char c : value) {
@@ -414,7 +415,6 @@ std::string VHiveTableWriter::_to_partition_value(const TypeDescriptor& type_des
414415
char buf[64];
415416
char* pos = value.to_string(buf);
416417
return std::string(buf, pos - buf - 1);
417-
break;
418418
}
419419
case TYPE_DATEV2: {
420420
DateV2Value<DateV2ValueType> value =

0 commit comments

Comments
 (0)