@@ -87,7 +87,8 @@ S3FileWriter::S3FileWriter(std::shared_ptr<Aws::S3::S3Client> client, std::strin
8787 : _path(fmt::format(" s3://{}/{}" , bucket, key)),
8888 _bucket (std::move(bucket)),
8989 _key(std::move(key)),
90- _client(std::move(client)) {
90+ _client(std::move(client)),
91+ _used_by_s3_committer(opts ? opts->used_by_s3_committer : false ) {
9192 s3_file_writer_total << 1 ;
9293 s3_file_being_written << 1 ;
9394 Aws::Http::SetCompliantRfc3986Encoding (true );
@@ -195,10 +196,7 @@ Status S3FileWriter::close() {
195196 Defer defer {[this ] { _closed = true ; }};
196197
197198 if (_upload_id.empty () && _pending_buf) {
198- // It might be one file less than 5MB, and call close without finalize
199- auto * buf = dynamic_cast <UploadFileBuffer*>(_pending_buf.get ());
200- DCHECK (buf != nullptr );
201- buf->set_upload_to_remote ([this ](UploadFileBuffer& b) { _put_object (b); });
199+ RETURN_IF_ERROR (_set_upload_to_remote_less_than_buffer_size ());
202200 }
203201
204202 if (_bytes_appended == 0 ) {
@@ -225,6 +223,13 @@ Status S3FileWriter::close() {
225223 RETURN_IF_ERROR (builder.build (&_pending_buf));
226224 auto * buf = dynamic_cast <UploadFileBuffer*>(_pending_buf.get ());
227225 DCHECK (buf != nullptr );
226+ if (_used_by_s3_committer) {
227+ buf->set_upload_to_remote ([part_num = _cur_part_num, this ](UploadFileBuffer& buf) {
228+ _upload_one_part (part_num, buf);
229+ });
230+ DCHECK (_cur_part_num == 1 );
231+ RETURN_IF_ERROR (_create_multi_upload_request ());
232+ }
228233 }
229234
230235 if (_pending_buf != nullptr ) {
@@ -392,56 +397,61 @@ Status S3FileWriter::_complete() {
392397 _wait_until_finish (" PutObject" );
393398 return _st;
394399 }
395- CompleteMultipartUploadRequest complete_request;
396- complete_request.WithBucket (_bucket).WithKey (_key).WithUploadId (_upload_id);
397-
400+ // Wait multipart load and finish.
398401 _wait_until_finish (" Complete" );
399402 DBUG_EXECUTE_IF (" s3_file_writer::_complete:1" , { _cur_part_num++; });
400- if (_failed || _completed_parts.size () != _cur_part_num) {
401- _st = Status::InternalError (
402- " error status {}, complete parts {}, cur part num {}, whole parts {}" , _st,
403- _completed_parts.size (), _cur_part_num, _dump_completed_part ());
404- LOG (WARNING ) << _st;
405- return _st;
406- }
407- // make sure _completed_parts are ascending order
408- std::sort (_completed_parts.begin (), _completed_parts.end (),
409- [](auto & p1, auto & p2) { return p1->GetPartNumber () < p2->GetPartNumber (); });
410- DBUG_EXECUTE_IF (" s3_file_writer::_complete:2" ,
411- { _completed_parts.back ()->SetPartNumber (10 * _completed_parts.size ()); });
412- CompletedMultipartUpload completed_upload;
413- for (size_t i = 0 ; i < _completed_parts.size (); i++) {
414- if (_completed_parts[i]->GetPartNumber () != i + 1 ) [[unlikely]] {
415- auto st = Status::InternalError (
416- " error status {}, part num not continous, expected num {}, actual num {}, "
417- " whole parts {}" ,
418- _st, i + 1 , _completed_parts[i]->GetPartNumber (), _dump_completed_part ());
419- LOG (WARNING ) << st;
420- _st = st;
421- return st;
403+ if (!_used_by_s3_committer) { // S3 committer will complete multipart upload file on FE side.
404+ CompleteMultipartUploadRequest complete_request;
405+ complete_request.WithBucket (_bucket).WithKey (_key).WithUploadId (_upload_id);
406+
407+ if (_failed || _completed_parts.size () != _cur_part_num) {
408+ _st = Status::InternalError (
409+ " error status {}, complete parts {}, cur part num {}, whole parts {}" , _st,
410+ _completed_parts.size (), _cur_part_num, _dump_completed_part ());
411+ LOG (WARNING ) << _st;
412+ return _st;
413+ }
414+ // make sure _completed_parts are ascending order
415+ std::sort (_completed_parts.begin (), _completed_parts.end (),
416+ [](auto & p1, auto & p2) { return p1->GetPartNumber () < p2->GetPartNumber (); });
417+ DBUG_EXECUTE_IF (" s3_file_writer::_complete:2" ,
418+ { _completed_parts.back ()->SetPartNumber (10 * _completed_parts.size ()); });
419+ CompletedMultipartUpload completed_upload;
420+ for (size_t i = 0 ; i < _completed_parts.size (); i++) {
421+ if (_completed_parts[i]->GetPartNumber () != i + 1 ) [[unlikely]] {
422+ auto st = Status::InternalError (
423+ " error status {}, part num not continous, expected num {}, actual num {}, "
424+ " whole parts {}" ,
425+ _st, i + 1 , _completed_parts[i]->GetPartNumber (), _dump_completed_part ());
426+ LOG (WARNING ) << st;
427+ _st = st;
428+ return st;
429+ }
430+ completed_upload.AddParts (*_completed_parts[i]);
422431 }
423- completed_upload.AddParts (*_completed_parts[i]);
424- }
425432
426- complete_request.WithMultipartUpload (completed_upload);
433+ complete_request.WithMultipartUpload (completed_upload);
427434
428- DBUG_EXECUTE_IF (" s3_file_writer::_complete:3" , {
429- auto s = Status::IOError (
430- " failed to create complete multi part upload (bucket={}, key={}): injected error" ,
431- _bucket, _path.native ());
432- LOG_WARNING (s.to_string ());
433- return s;
434- });
435- SCOPED_BVAR_LATENCY (s3_bvar::s3_multi_part_upload_latency);
436- auto complete_outcome = _client->CompleteMultipartUpload (complete_request);
435+ DBUG_EXECUTE_IF (" s3_file_writer::_complete:3" , {
436+ auto s = Status::IOError (
437+ " failed to create complete multi part upload (bucket={}, key={}): injected "
438+ " error" ,
439+ _bucket, _path.native ());
440+ LOG_WARNING (s.to_string ());
441+ return s;
442+ });
443+ SCOPED_BVAR_LATENCY (s3_bvar::s3_multi_part_upload_latency);
444+ auto complete_outcome = _client->CompleteMultipartUpload (complete_request);
437445
438- if (!complete_outcome.IsSuccess ()) {
439- _st = s3fs_error (
440- complete_outcome.GetError (),
441- fmt::format (" failed to complete multi part upload {}, upload_id={}, whole parts={}" ,
446+ if (!complete_outcome.IsSuccess ()) {
447+ _st = s3fs_error (
448+ complete_outcome.GetError (),
449+ fmt::format (
450+ " failed to complete multi part upload {}, upload_id={}, whole parts={}" ,
442451 _path.native (), _upload_id, _dump_completed_part ()));
443- LOG (WARNING ) << _st;
444- return _st;
452+ LOG (WARNING ) << _st;
453+ return _st;
454+ }
445455 }
446456 s3_file_created_total << 1 ;
447457 return Status::OK ();
@@ -457,12 +467,8 @@ Status S3FileWriter::finalize() {
457467 // submit pending buf if it's not nullptr
458468 // it's the last buf, we can submit it right now
459469 if (_pending_buf != nullptr ) {
460- // if we only need to upload one file less than 5MB, we can just
461- // call PutObject to reduce the network IO
462470 if (_upload_id.empty ()) {
463- auto * buf = dynamic_cast <UploadFileBuffer*>(_pending_buf.get ());
464- DCHECK (buf != nullptr );
465- buf->set_upload_to_remote ([this ](UploadFileBuffer& b) { _put_object (b); });
471+ RETURN_IF_ERROR (_set_upload_to_remote_less_than_buffer_size ());
466472 }
467473 _countdown_event.add_count ();
468474 RETURN_IF_ERROR (_pending_buf->submit (std::move (_pending_buf)));
@@ -472,6 +478,24 @@ Status S3FileWriter::finalize() {
472478 return _st;
473479}
474480
481+ Status S3FileWriter::_set_upload_to_remote_less_than_buffer_size () {
482+ auto * buf = dynamic_cast <UploadFileBuffer*>(_pending_buf.get ());
483+ DCHECK (buf != nullptr );
484+ if (_used_by_s3_committer) {
485+ // If used_by_s3_committer, we always use multi-parts uploading.
486+ buf->set_upload_to_remote ([part_num = _cur_part_num, this ](UploadFileBuffer& buf) {
487+ _upload_one_part (part_num, buf);
488+ });
489+ DCHECK (_cur_part_num == 1 );
490+ RETURN_IF_ERROR (_create_multi_upload_request ());
491+ } else {
492+ // if we only need to upload one file less than 5MB, we can just
493+ // call PutObject to reduce the network IO
494+ buf->set_upload_to_remote ([this ](UploadFileBuffer& b) { _put_object (b); });
495+ }
496+ return Status::OK ();
497+ }
498+
475499void S3FileWriter::_put_object (UploadFileBuffer& buf) {
476500 DCHECK (!closed ());
477501 Aws::S3 ::Model::PutObjectRequest request;
0 commit comments