Skip to content

Commit f45a4a5

Browse files
liaoxin01wyxxxcat
authored andcommitted
[opt](recycler) Do not retry S3 throttling at the SDK layer (apache#9676)
Follow-up to apache#9650 / apache#9656. Let the recycler's group-level retry be the primary handler for S3 per-prefix throttling, instead of the AWS SDK retrying it internally. The recycler's S3 client used `S3CustomRetryStrategy(max_s3_client_retry)` (default 10), which retries `503 SlowDown` / `429` / `509` internally with un-jittered, closely-spaced exponential backoff. This: - hammers the same hot prefix in lockstep (no jitter) — what AWS explicitly warns against for `SlowDown`; - largely pre-empts the group-level jittered backoff added in apache#9650, so under mild/medium throttling the group retry never engages. - Add a recycler-only `RecyclerS3RetryStrategy` that returns `false` for throttling codes (503/429/509), so a throttle is surfaced immediately and handled by the group-level retry. Other transient errors keep the default retry behavior. - Applied only to the recycler's S3 client; BE keeps `S3CustomRetryStrategy`. Affects all recycler S3 operations on that client, not just rowset deletes. Deletes have the group-level retry as a fallback; other ops rely on the next recycle cycle. Azure's retry path is unchanged.
1 parent 2f5cef3 commit f45a4a5

4 files changed

Lines changed: 54 additions & 21 deletions

File tree

‎cloud/src/recycler/s3_accessor.cpp‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,10 @@
2121
#include <aws/core/auth/AWSCredentials.h>
2222
#include <aws/core/auth/AWSCredentialsProviderChain.h>
2323
#include <aws/core/auth/STSCredentialsProvider.h>
24+
#include <aws/core/client/AWSError.h>
25+
#include <aws/core/client/CoreErrors.h>
2426
#include <aws/core/client/DefaultRetryStrategy.h>
27+
#include <aws/core/http/HttpResponse.h>
2528
#include <aws/core/platform/Environment.h>
2629
#include <aws/identity-management/auth/STSAssumeRoleCredentialsProvider.h>
2730
#include <aws/s3/S3Client.h>
@@ -78,6 +81,28 @@ bvar::Adder<int64_t> get_rate_limit_exceed_req_num("get_rate_limit_exceed_req_nu
7881
bvar::Adder<int64_t> put_rate_limit_ns("put_rate_limit_ns");
7982
bvar::Adder<int64_t> put_rate_limit_exceed_req_num("put_rate_limit_exceed_req_num");
8083

84+
namespace {
85+
// Retry strategy for the recycler's S3 client. It behaves like S3CustomRetryStrategy
86+
// (same retry decisions, metrics and logging) except it does NOT retry throttling
87+
// errors (503 SlowDown / 429 / 509) at the SDK layer: those are surfaced immediately
88+
// so the recycler's group-level retry can back off with jitter per group instead of
89+
// the SDK hammering the same hot prefix with un-jittered, closely-spaced retries.
90+
class RecyclerS3RetryStrategy final : public doris::S3CustomRetryStrategy {
91+
public:
92+
explicit RecyclerS3RetryStrategy(int max_retries) : doris::S3CustomRetryStrategy(max_retries) {}
93+
94+
bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error,
95+
long attempted_retries) const override {
96+
// Throttling is surfaced immediately for the group-level retry; everything
97+
// else keeps S3CustomRetryStrategy's behavior (retry decision, metrics, logs).
98+
if (is_s3_throttle_error(error)) {
99+
return false;
100+
}
101+
return doris::S3CustomRetryStrategy::ShouldRetry(error, attempted_retries);
102+
}
103+
};
104+
} // namespace
105+
81106
AccessorRateLimiter::AccessorRateLimiter()
82107
: _rate_limiters(
83108
{std::make_unique<S3RateLimiterHolder>(
@@ -441,8 +466,10 @@ int S3Accessor::init() {
441466
if (config::s3_client_http_scheme == "http") {
442467
aws_config.scheme = Aws::Http::Scheme::HTTP;
443468
}
444-
aws_config.retryStrategy = std::make_shared<S3CustomRetryStrategy>(
445-
config::max_s3_client_retry, config::s3_client_retry_slow_down);
469+
// Throttling (503 SlowDown / 429 / 509) is intentionally not retried at the
470+
// SDK layer here; it bubbles up to the recycler's group-level jittered retry.
471+
aws_config.retryStrategy =
472+
std::make_shared<RecyclerS3RetryStrategy>(config::max_s3_client_retry);
446473

447474
if (_ca_cert_file_path.empty()) {
448475
_ca_cert_file_path =

‎cloud/src/recycler/s3_accessor.h‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#pragma once
1919

2020
#include <aws/core/Aws.h>
21+
#include <aws/core/http/HttpResponse.h>
2122
#include <bvar/latency_recorder.h>
2223

2324
#include <array>
@@ -41,6 +42,28 @@ namespace cloud {
4142
class ObjectStoreInfoPB;
4243
class SimpleThreadPool;
4344

45+
// Returns true if the AWS error indicates the request was throttled (rate limited)
46+
// by the backend, e.g. 503 SlowDown / 429 TooManyRequests / 509. Templated so it
47+
// works for both Aws::S3::S3Error and Aws::Client::AWSError<CoreErrors>; both expose
48+
// GetResponseCode() and GetExceptionName(). This is the single source of truth used
49+
// by the S3 retry strategy (whether to retry at the SDK layer) and by the delete
50+
// paths (whether to surface ObjectStorageResponse::THROTTLED).
51+
template <typename AwsErrorType>
52+
inline bool is_s3_throttle_error(const AwsErrorType& error) {
53+
switch (error.GetResponseCode()) {
54+
case Aws::Http::HttpResponseCode::SERVICE_UNAVAILABLE: // 503, e.g. SlowDown
55+
case Aws::Http::HttpResponseCode::TOO_MANY_REQUESTS: // 429
56+
case Aws::Http::HttpResponseCode::BANDWIDTH_LIMIT_EXCEEDED: // 509
57+
return true;
58+
default:
59+
break;
60+
}
61+
const auto& name = error.GetExceptionName();
62+
return name == "SlowDown" || name == "RequestThrottled" ||
63+
name == "RequestThrottledException" || name == "ThrottlingException" ||
64+
name == "Throttling" || name == "TooManyRequests" || name == "RequestLimitExceeded";
65+
}
66+
4467
namespace s3_bvar {
4568
extern bvar::LatencyRecorder s3_get_latency;
4669
extern bvar::LatencyRecorder s3_put_latency;

‎cloud/src/recycler/s3_obj_client.cpp‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -44,24 +44,6 @@ namespace doris::cloud {
4444
return {Aws::S3::S3Errors::INTERNAL_FAILURE, "exceeds limit", "exceeds limit", false};
4545
}
4646

47-
// Returns true if the S3 error indicates the request was throttled (rate limited)
48-
// by the backend, e.g. 503 SlowDown / 429 TooManyRequests. The caller can then
49-
// back off and retry rather than treating the failure as permanent.
50-
static bool is_s3_throttle_error(const Aws::S3::S3Error& error) {
51-
switch (error.GetResponseCode()) {
52-
case Aws::Http::HttpResponseCode::SERVICE_UNAVAILABLE: // 503, e.g. SlowDown
53-
case Aws::Http::HttpResponseCode::TOO_MANY_REQUESTS: // 429
54-
case Aws::Http::HttpResponseCode::BANDWIDTH_LIMIT_EXCEEDED: // 509
55-
return true;
56-
default:
57-
break;
58-
}
59-
const auto& name = error.GetExceptionName();
60-
return name == "SlowDown" || name == "RequestThrottled" ||
61-
name == "RequestThrottledException" || name == "ThrottlingException" ||
62-
name == "Throttling" || name == "TooManyRequests" || name == "RequestLimitExceeded";
63-
}
64-
6547
template <typename Func>
6648
auto s3_rate_limit(S3RateLimitType op, Func callback) -> decltype(callback()) {
6749
using T = decltype(callback());

‎common/cpp/obj_retry_strategy.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@
2525
#endif
2626

2727
namespace doris {
28-
class S3CustomRetryStrategy final : public Aws::Client::DefaultRetryStrategy {
28+
// Not `final`: the cloud recycler derives a variant that skips retrying throttling.
29+
class S3CustomRetryStrategy : public Aws::Client::DefaultRetryStrategy {
2930
public:
3031
S3CustomRetryStrategy(int maxRetries, bool retry_slow_down = true);
3132
~S3CustomRetryStrategy() override;

0 commit comments

Comments
 (0)