Skip to content

Commit a2376e9

Browse files
kaka11chenHappenLee
authored andcommitted
[Feature](hive-writer) Implements s3 file committer. (apache#33937)
Issue Number: apache#31442 [Feature] (hive-writer) Implements s3 file committer. S3 committer will start multipart uploading all files on BE side, and then complete multipart upload these files on FE side. If you do not complete multi parts of a file, the file will not be visible. So in this way, the atomicity of a single file can be guaranteed. But it still cannot guarantee the atomicity of multiple files. Because hive committers have best-effort semantics, this shortens the inconsistent time window. ## ChangeList: - Add `used_by_s3_committer` in `FileWriterOptions` on BE side to start multi-part uploading files, then complete multi-part uploading files on FE side. - `cosn://`use s3 client on FE side, because it need to complete multi-part uploading files on FE side. - Add `Status directoryExists(String dir)` and `Status deleteDirectory` in `FileSystem`.
1 parent 61714d9 commit a2376e9

33 files changed

Lines changed: 980 additions & 190 deletions

‎be/src/io/fs/file_writer.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,14 @@ struct FileCacheAllocatorBuilder;
3232

3333
// Only affects remote file writers
3434
struct FileWriterOptions {
35+
// S3 committer will start multipart uploading all files on BE side,
36+
// and then complete multipart upload these files on FE side.
37+
// If you do not complete multi parts of a file, the file will not be visible.
38+
// So in this way, the atomicity of a single file can be guaranteed. But it still cannot
39+
// guarantee the atomicity of multiple files.
40+
// Because hive committers have best-effort semantics,
41+
// this shortens the inconsistent time window.
42+
bool used_by_s3_committer = false;
3543
bool write_file_cache = false;
3644
bool is_cold_data = false;
3745
bool sync_file_data = true; // Whether flush data into storage system

‎be/src/io/fs/s3_file_writer.cpp‎

Lines changed: 77 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,8 @@ S3FileWriter::S3FileWriter(std::shared_ptr<Aws::S3::S3Client> client, std::strin
8787
: _path(fmt::format("s3://{}/{}", bucket, key)),
8888
_bucket(std::move(bucket)),
8989
_key(std::move(key)),
90-
_client(std::move(client)) {
90+
_client(std::move(client)),
91+
_used_by_s3_committer(opts ? opts->used_by_s3_committer : false) {
9192
s3_file_writer_total << 1;
9293
s3_file_being_written << 1;
9394
Aws::Http::SetCompliantRfc3986Encoding(true);
@@ -195,10 +196,7 @@ Status S3FileWriter::close() {
195196
Defer defer {[this] { _closed = true; }};
196197

197198
if (_upload_id.empty() && _pending_buf) {
198-
// It might be one file less than 5MB, and call close without finalize
199-
auto* buf = dynamic_cast<UploadFileBuffer*>(_pending_buf.get());
200-
DCHECK(buf != nullptr);
201-
buf->set_upload_to_remote([this](UploadFileBuffer& b) { _put_object(b); });
199+
RETURN_IF_ERROR(_set_upload_to_remote_less_than_buffer_size());
202200
}
203201

204202
if (_bytes_appended == 0) {
@@ -225,6 +223,13 @@ Status S3FileWriter::close() {
225223
RETURN_IF_ERROR(builder.build(&_pending_buf));
226224
auto* buf = dynamic_cast<UploadFileBuffer*>(_pending_buf.get());
227225
DCHECK(buf != nullptr);
226+
if (_used_by_s3_committer) {
227+
buf->set_upload_to_remote([part_num = _cur_part_num, this](UploadFileBuffer& buf) {
228+
_upload_one_part(part_num, buf);
229+
});
230+
DCHECK(_cur_part_num == 1);
231+
RETURN_IF_ERROR(_create_multi_upload_request());
232+
}
228233
}
229234

230235
if (_pending_buf != nullptr) {
@@ -392,56 +397,61 @@ Status S3FileWriter::_complete() {
392397
_wait_until_finish("PutObject");
393398
return _st;
394399
}
395-
CompleteMultipartUploadRequest complete_request;
396-
complete_request.WithBucket(_bucket).WithKey(_key).WithUploadId(_upload_id);
397-
400+
// Wait multipart load and finish.
398401
_wait_until_finish("Complete");
399402
DBUG_EXECUTE_IF("s3_file_writer::_complete:1", { _cur_part_num++; });
400-
if (_failed || _completed_parts.size() != _cur_part_num) {
401-
_st = Status::InternalError(
402-
"error status {}, complete parts {}, cur part num {}, whole parts {}", _st,
403-
_completed_parts.size(), _cur_part_num, _dump_completed_part());
404-
LOG(WARNING) << _st;
405-
return _st;
406-
}
407-
// make sure _completed_parts are ascending order
408-
std::sort(_completed_parts.begin(), _completed_parts.end(),
409-
[](auto& p1, auto& p2) { return p1->GetPartNumber() < p2->GetPartNumber(); });
410-
DBUG_EXECUTE_IF("s3_file_writer::_complete:2",
411-
{ _completed_parts.back()->SetPartNumber(10 * _completed_parts.size()); });
412-
CompletedMultipartUpload completed_upload;
413-
for (size_t i = 0; i < _completed_parts.size(); i++) {
414-
if (_completed_parts[i]->GetPartNumber() != i + 1) [[unlikely]] {
415-
auto st = Status::InternalError(
416-
"error status {}, part num not continous, expected num {}, actual num {}, "
417-
"whole parts {}",
418-
_st, i + 1, _completed_parts[i]->GetPartNumber(), _dump_completed_part());
419-
LOG(WARNING) << st;
420-
_st = st;
421-
return st;
403+
if (!_used_by_s3_committer) { // S3 committer will complete multipart upload file on FE side.
404+
CompleteMultipartUploadRequest complete_request;
405+
complete_request.WithBucket(_bucket).WithKey(_key).WithUploadId(_upload_id);
406+
407+
if (_failed || _completed_parts.size() != _cur_part_num) {
408+
_st = Status::InternalError(
409+
"error status {}, complete parts {}, cur part num {}, whole parts {}", _st,
410+
_completed_parts.size(), _cur_part_num, _dump_completed_part());
411+
LOG(WARNING) << _st;
412+
return _st;
413+
}
414+
// make sure _completed_parts are ascending order
415+
std::sort(_completed_parts.begin(), _completed_parts.end(),
416+
[](auto& p1, auto& p2) { return p1->GetPartNumber() < p2->GetPartNumber(); });
417+
DBUG_EXECUTE_IF("s3_file_writer::_complete:2",
418+
{ _completed_parts.back()->SetPartNumber(10 * _completed_parts.size()); });
419+
CompletedMultipartUpload completed_upload;
420+
for (size_t i = 0; i < _completed_parts.size(); i++) {
421+
if (_completed_parts[i]->GetPartNumber() != i + 1) [[unlikely]] {
422+
auto st = Status::InternalError(
423+
"error status {}, part num not continous, expected num {}, actual num {}, "
424+
"whole parts {}",
425+
_st, i + 1, _completed_parts[i]->GetPartNumber(), _dump_completed_part());
426+
LOG(WARNING) << st;
427+
_st = st;
428+
return st;
429+
}
430+
completed_upload.AddParts(*_completed_parts[i]);
422431
}
423-
completed_upload.AddParts(*_completed_parts[i]);
424-
}
425432

426-
complete_request.WithMultipartUpload(completed_upload);
433+
complete_request.WithMultipartUpload(completed_upload);
427434

428-
DBUG_EXECUTE_IF("s3_file_writer::_complete:3", {
429-
auto s = Status::IOError(
430-
"failed to create complete multi part upload (bucket={}, key={}): injected error",
431-
_bucket, _path.native());
432-
LOG_WARNING(s.to_string());
433-
return s;
434-
});
435-
SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency);
436-
auto complete_outcome = _client->CompleteMultipartUpload(complete_request);
435+
DBUG_EXECUTE_IF("s3_file_writer::_complete:3", {
436+
auto s = Status::IOError(
437+
"failed to create complete multi part upload (bucket={}, key={}): injected "
438+
"error",
439+
_bucket, _path.native());
440+
LOG_WARNING(s.to_string());
441+
return s;
442+
});
443+
SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency);
444+
auto complete_outcome = _client->CompleteMultipartUpload(complete_request);
437445

438-
if (!complete_outcome.IsSuccess()) {
439-
_st = s3fs_error(
440-
complete_outcome.GetError(),
441-
fmt::format("failed to complete multi part upload {}, upload_id={}, whole parts={}",
446+
if (!complete_outcome.IsSuccess()) {
447+
_st = s3fs_error(
448+
complete_outcome.GetError(),
449+
fmt::format(
450+
"failed to complete multi part upload {}, upload_id={}, whole parts={}",
442451
_path.native(), _upload_id, _dump_completed_part()));
443-
LOG(WARNING) << _st;
444-
return _st;
452+
LOG(WARNING) << _st;
453+
return _st;
454+
}
445455
}
446456
s3_file_created_total << 1;
447457
return Status::OK();
@@ -457,12 +467,8 @@ Status S3FileWriter::finalize() {
457467
// submit pending buf if it's not nullptr
458468
// it's the last buf, we can submit it right now
459469
if (_pending_buf != nullptr) {
460-
// if we only need to upload one file less than 5MB, we can just
461-
// call PutObject to reduce the network IO
462470
if (_upload_id.empty()) {
463-
auto* buf = dynamic_cast<UploadFileBuffer*>(_pending_buf.get());
464-
DCHECK(buf != nullptr);
465-
buf->set_upload_to_remote([this](UploadFileBuffer& b) { _put_object(b); });
471+
RETURN_IF_ERROR(_set_upload_to_remote_less_than_buffer_size());
466472
}
467473
_countdown_event.add_count();
468474
RETURN_IF_ERROR(_pending_buf->submit(std::move(_pending_buf)));
@@ -472,6 +478,24 @@ Status S3FileWriter::finalize() {
472478
return _st;
473479
}
474480

481+
Status S3FileWriter::_set_upload_to_remote_less_than_buffer_size() {
482+
auto* buf = dynamic_cast<UploadFileBuffer*>(_pending_buf.get());
483+
DCHECK(buf != nullptr);
484+
if (_used_by_s3_committer) {
485+
// If used_by_s3_committer, we always use multi-parts uploading.
486+
buf->set_upload_to_remote([part_num = _cur_part_num, this](UploadFileBuffer& buf) {
487+
_upload_one_part(part_num, buf);
488+
});
489+
DCHECK(_cur_part_num == 1);
490+
RETURN_IF_ERROR(_create_multi_upload_request());
491+
} else {
492+
// if we only need to upload one file less than 5MB, we can just
493+
// call PutObject to reduce the network IO
494+
buf->set_upload_to_remote([this](UploadFileBuffer& b) { _put_object(b); });
495+
}
496+
return Status::OK();
497+
}
498+
475499
void S3FileWriter::_put_object(UploadFileBuffer& buf) {
476500
DCHECK(!closed());
477501
Aws::S3::Model::PutObjectRequest request;

‎be/src/io/fs/s3_file_writer.h‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,12 +62,21 @@ class S3FileWriter final : public FileWriter {
6262
return _cache_builder == nullptr ? nullptr : _cache_builder.get();
6363
}
6464

65+
const std::vector<std::unique_ptr<Aws::S3::Model::CompletedPart>>& completed_parts() const {
66+
return _completed_parts;
67+
}
68+
69+
const std::string& key() const { return _key; }
70+
const std::string& bucket() const { return _bucket; }
71+
const std::string& upload_id() const { return _upload_id; }
72+
6573
private:
6674
Status _abort();
6775
[[nodiscard]] std::string _dump_completed_part() const;
6876
void _wait_until_finish(std::string_view task_name);
6977
Status _complete();
7078
Status _create_multi_upload_request();
79+
Status _set_upload_to_remote_less_than_buffer_size();
7180
void _put_object(UploadFileBuffer& buf);
7281
void _upload_one_part(int64_t part_num, UploadFileBuffer& buf);
7382

@@ -95,6 +104,15 @@ class S3FileWriter final : public FileWriter {
95104
std::shared_ptr<FileBuffer> _pending_buf;
96105
std::unique_ptr<FileCacheAllocatorBuilder>
97106
_cache_builder; // nullptr if disable write file cache
107+
108+
// S3 committer will start multipart uploading all files on BE side,
109+
// and then complete multipart upload these files on FE side.
110+
// If you do not complete multi parts of a file, the file will not be visible.
111+
// So in this way, the atomicity of a single file can be guaranteed. But it still cannot
112+
// guarantee the atomicity of multiple files.
113+
// Because hive committers have best-effort semantics,
114+
// this shortens the inconsistent time window.
115+
bool _used_by_s3_committer;
98116
};
99117

100118
} // namespace io

‎be/src/vec/sink/writer/vhive_partition_writer.cpp‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,10 @@
1717

1818
#include "vhive_partition_writer.h"
1919

20+
#include <aws/s3/model/CompletedPart.h>
21+
2022
#include "io/file_factory.h"
23+
#include "io/fs/s3_file_writer.h"
2124
#include "runtime/runtime_state.h"
2225
#include "vec/columns/column_map.h"
2326
#include "vec/core/materialize_block.h"
@@ -55,7 +58,8 @@ Status VHivePartitionWriter::open(RuntimeState* state, RuntimeProfile* profile)
5558
io::FileDescription file_description = {
5659
.path = fmt::format("{}/{}", _write_info.write_path, _get_target_file_name())};
5760
_fs = DORIS_TRY(FileFactory::create_fs(fs_properties, file_description));
58-
RETURN_IF_ERROR(_fs->create_file(file_description.path, &_file_writer));
61+
io::FileWriterOptions file_writer_options = {.used_by_s3_committer = true};
62+
RETURN_IF_ERROR(_fs->create_file(file_description.path, &_file_writer, &file_writer_options));
5963

6064
std::vector<std::string> column_names;
6165
column_names.reserve(_columns.size());
@@ -189,12 +193,28 @@ THivePartitionUpdate VHivePartitionWriter::_build_partition_update() {
189193
hive_partition_update.__set_name(_partition_name);
190194
hive_partition_update.__set_update_mode(_update_mode);
191195
THiveLocationParams location;
192-
location.__set_write_path(_write_info.write_path);
196+
location.__set_write_path(_write_info.original_write_path);
193197
location.__set_target_path(_write_info.target_path);
194198
hive_partition_update.__set_location(location);
195199
hive_partition_update.__set_file_names({_get_target_file_name()});
196200
hive_partition_update.__set_row_count(_row_count);
197201
hive_partition_update.__set_file_size(_input_size_in_bytes);
202+
203+
if (_write_info.file_type == TFileType::FILE_S3) {
204+
doris::io::S3FileWriter* s3_mpu_file_writer =
205+
dynamic_cast<doris::io::S3FileWriter*>(_file_writer.get());
206+
TS3MPUPendingUpload s3_mpu_pending_upload;
207+
s3_mpu_pending_upload.__set_bucket(s3_mpu_file_writer->bucket());
208+
s3_mpu_pending_upload.__set_key(s3_mpu_file_writer->key());
209+
s3_mpu_pending_upload.__set_upload_id(s3_mpu_file_writer->upload_id());
210+
211+
std::map<int, std::string> etags;
212+
for (auto& completed_part : s3_mpu_file_writer->completed_parts()) {
213+
etags.insert({completed_part->GetPartNumber(), completed_part->GetETag()});
214+
}
215+
s3_mpu_pending_upload.__set_etags(etags);
216+
hive_partition_update.__set_s3_mpu_pending_uploads({s3_mpu_pending_upload});
217+
}
198218
return hive_partition_update;
199219
}
200220

‎be/src/vec/sink/writer/vhive_partition_writer.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ class VHivePartitionWriter {
4343
public:
4444
struct WriteInfo {
4545
std::string write_path;
46+
std::string original_write_path;
4647
std::string target_path;
4748
TFileType::type file_type;
4849
};

‎be/src/vec/sink/writer/vhive_table_writer.cpp‎

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -256,26 +256,30 @@ std::shared_ptr<VHivePartitionWriter> VHiveTableWriter::_create_partition_writer
256256
if (existing_table == false) { // new table
257257
update_mode = TUpdateMode::NEW;
258258
if (_partition_columns_input_index.empty()) { // new unpartitioned table
259-
write_info = {write_location.write_path, write_location.target_path,
260-
write_location.file_type};
259+
write_info = {write_location.write_path, write_location.original_write_path,
260+
write_location.target_path, write_location.file_type};
261261
} else { // a new partition in a new partitioned table
262262
auto write_path = fmt::format("{}/{}", write_location.write_path, partition_name);
263+
auto original_write_path =
264+
fmt::format("{}/{}", write_location.original_write_path, partition_name);
263265
auto target_path = fmt::format("{}/{}", write_location.target_path, partition_name);
264-
write_info = {std::move(write_path), std::move(target_path),
265-
write_location.file_type};
266+
write_info = {std::move(write_path), std::move(original_write_path),
267+
std::move(target_path), write_location.file_type};
266268
}
267269
} else { // a new partition in an existing partitioned table, or an existing unpartitioned table
268270
if (_partition_columns_input_index.empty()) { // an existing unpartitioned table
269271
update_mode =
270272
!hive_table_sink.overwrite ? TUpdateMode::APPEND : TUpdateMode::OVERWRITE;
271-
write_info = {write_location.write_path, write_location.target_path,
272-
write_location.file_type};
273+
write_info = {write_location.write_path, write_location.original_write_path,
274+
write_location.target_path, write_location.file_type};
273275
} else { // a new partition in an existing partitioned table
274276
update_mode = TUpdateMode::NEW;
275277
auto write_path = fmt::format("{}/{}", write_location.write_path, partition_name);
278+
auto original_write_path =
279+
fmt::format("{}/{}", write_location.original_write_path, partition_name);
276280
auto target_path = fmt::format("{}/{}", write_location.target_path, partition_name);
277-
write_info = {std::move(write_path), std::move(target_path),
278-
write_location.file_type};
281+
write_info = {std::move(write_path), std::move(original_write_path),
282+
std::move(target_path), write_location.file_type};
279283
}
280284
// need to get schema from existing table ?
281285
}
@@ -285,16 +289,21 @@ std::shared_ptr<VHivePartitionWriter> VHiveTableWriter::_create_partition_writer
285289
if (!hive_table_sink.overwrite) {
286290
update_mode = TUpdateMode::APPEND;
287291
auto write_path = fmt::format("{}/{}", write_location.write_path, partition_name);
292+
auto original_write_path =
293+
fmt::format("{}/{}", write_location.original_write_path, partition_name);
288294
auto target_path = fmt::format("{}", existing_partition->location.target_path);
289-
write_info = {std::move(write_path), std::move(target_path),
290-
existing_partition->location.file_type};
295+
write_info = {std::move(write_path), std::move(original_write_path),
296+
std::move(target_path), existing_partition->location.file_type};
291297
file_format_type = existing_partition->file_format;
292298
write_compress_type = hive_table_sink.compression_type;
293299
} else {
294300
update_mode = TUpdateMode::OVERWRITE;
295301
auto write_path = fmt::format("{}/{}", write_location.write_path, partition_name);
302+
auto original_write_path =
303+
fmt::format("{}/{}", write_location.original_write_path, partition_name);
296304
auto target_path = fmt::format("{}/{}", write_location.target_path, partition_name);
297-
write_info = {std::move(write_path), std::move(target_path), write_location.file_type};
305+
write_info = {std::move(write_path), std::move(original_write_path),
306+
std::move(target_path), write_location.file_type};
298307
file_format_type = hive_table_sink.file_format;
299308
write_compress_type = hive_table_sink.compression_type;
300309
// need to get schema from existing table ?

0 commit comments

Comments
 (0)