|
21 | 21 | #include <aws/core/auth/AWSCredentials.h> |
22 | 22 | #include <aws/core/auth/AWSCredentialsProviderChain.h> |
23 | 23 | #include <aws/core/auth/STSCredentialsProvider.h> |
| 24 | +#include <aws/core/client/AWSError.h> |
| 25 | +#include <aws/core/client/CoreErrors.h> |
24 | 26 | #include <aws/core/client/DefaultRetryStrategy.h> |
| 27 | +#include <aws/core/http/HttpResponse.h> |
25 | 28 | #include <aws/core/platform/Environment.h> |
26 | 29 | #include <aws/identity-management/auth/STSAssumeRoleCredentialsProvider.h> |
27 | 30 | #include <aws/s3/S3Client.h> |
@@ -78,6 +81,28 @@ bvar::Adder<int64_t> get_rate_limit_exceed_req_num("get_rate_limit_exceed_req_nu |
78 | 81 | bvar::Adder<int64_t> put_rate_limit_ns("put_rate_limit_ns"); |
79 | 82 | bvar::Adder<int64_t> put_rate_limit_exceed_req_num("put_rate_limit_exceed_req_num"); |
80 | 83 |
|
| 84 | +namespace { |
| 85 | +// Retry strategy for the recycler's S3 client. It behaves like S3CustomRetryStrategy |
| 86 | +// (same retry decisions, metrics and logging) except it does NOT retry throttling |
| 87 | +// errors (503 SlowDown / 429 / 509) at the SDK layer: those are surfaced immediately |
| 88 | +// so the recycler's group-level retry can back off with jitter per group instead of |
| 89 | +// the SDK hammering the same hot prefix with un-jittered, closely-spaced retries. |
| 90 | +class RecyclerS3RetryStrategy final : public doris::S3CustomRetryStrategy { |
| 91 | +public: |
| 92 | + explicit RecyclerS3RetryStrategy(int max_retries) : doris::S3CustomRetryStrategy(max_retries) {} |
| 93 | + |
| 94 | + bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, |
| 95 | + long attempted_retries) const override { |
| 96 | + // Throttling is surfaced immediately for the group-level retry; everything |
| 97 | + // else keeps S3CustomRetryStrategy's behavior (retry decision, metrics, logs). |
| 98 | + if (is_s3_throttle_error(error)) { |
| 99 | + return false; |
| 100 | + } |
| 101 | + return doris::S3CustomRetryStrategy::ShouldRetry(error, attempted_retries); |
| 102 | + } |
| 103 | +}; |
| 104 | +} // namespace |
| 105 | + |
81 | 106 | AccessorRateLimiter::AccessorRateLimiter() |
82 | 107 | : _rate_limiters( |
83 | 108 | {std::make_unique<S3RateLimiterHolder>( |
@@ -441,8 +466,10 @@ int S3Accessor::init() { |
441 | 466 | if (config::s3_client_http_scheme == "http") { |
442 | 467 | aws_config.scheme = Aws::Http::Scheme::HTTP; |
443 | 468 | } |
444 | | - aws_config.retryStrategy = std::make_shared<S3CustomRetryStrategy>( |
445 | | - config::max_s3_client_retry, config::s3_client_retry_slow_down); |
| 469 | + // Throttling (503 SlowDown / 429 / 509) is intentionally not retried at the |
| 470 | + // SDK layer here; it bubbles up to the recycler's group-level jittered retry. |
| 471 | + aws_config.retryStrategy = |
| 472 | + std::make_shared<RecyclerS3RetryStrategy>(config::max_s3_client_retry); |
446 | 473 |
|
447 | 474 | if (_ca_cert_file_path.empty()) { |
448 | 475 | _ca_cert_file_path = |
|
0 commit comments