Skip to content

Commit 61f6dec

Browse files
committed
[refactor](be) Rename S3ClientFactory to AwsSdkManager and move the Azure parts out of it
### What problem does this PR solve? Issue Number: None Related PR: apache#69605 Problem Summary: After client construction moved to `ObjClientFactory`, `S3ClientFactory` no longer built clients, yet its name still said so, and it still carried two Azure parts: its constructor set up the Azure SDK log, and `AzureClientProvider` called `S3ClientFactory::instance().ca_cert_file_path()` for its CA bundle. Review pointed out both. - `S3ClientFactory` becomes `AwsSdkManager`. It keeps only the AWS SDK lifetime (`InitAPI` / `ShutdownAPI`), `getClientConfiguration` and `create_aws_credentials_provider`, which the S3 provider and Kinesis use. gtest suite and file names stay. - The Azure SDK log setup moves unchanged into `init_azure_sdk_logging()` in `azure_client_provider`. `ObjClientFactory` calls it on construction, right after initializing the AWS SDK, and BE startup now constructs `ObjClientFactory` instead of `S3ClientFactory`, so both SDKs are still set up at startup in the same thread, and the AWS SDK is still shut down after the providers release their clients. - The CA bundle cache moves unchanged into `io/obj_storage/obj_client_ca.{h,cpp}` (`obj_client_ca_cert_file_path()`): the first valid path of `ca_cert_file_paths`, cached, looked up again while empty. Both providers and the AWS credentials provider use it, so the Azure provider no longer depends on an AWS class. `AwsSdkManager` still resolves it once on construction. The CA test uses a test hook instead of the removed private members. ### Release note None ### Check List (For Author) - Test: Unit Test (`S3ClientFactoryTest.RefreshCaCertForCredentialsProvider` now goes through the new hook). Not compiled locally; CI compiles and runs the unit tests. Build hygiene and clang-format 16 passed. - Behavior changed: No - Does this need documentation: No
1 parent 583866c commit 61f6dec

16 files changed

Lines changed: 182 additions & 110 deletions

‎be/src/io/obj_storage/azure_client_provider.cpp‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,11 @@
2323

2424
#include "common/config.h"
2525
#include "common/logging.h"
26+
#include "io/obj_storage/obj_client_ca.h"
2627
#include "io/obj_storage/obj_client_rate_limit.h"
2728

2829
#ifdef USE_AZURE
30+
#include <azure/core/diagnostics/logger.hpp>
2931
#include <azure/core/http/curl_transport.hpp>
3032
#include <azure/storage/blobs/blob_container_client.hpp>
3133
#include <cstdlib>
@@ -36,7 +38,6 @@
3638
#include "cpp/obj-client/azure_obj_storage_client.h"
3739
#include "cpp/obj_retry_strategy.h"
3840
#include "cpp/util.h"
39-
#include "util/s3_util.h"
4041
#endif
4142

4243
namespace doris::io {
@@ -72,6 +73,35 @@ std::string build_azure_tls_debug_context(const std::string& selected_ca_file) {
7273

7374
} // namespace
7475

76+
void init_azure_sdk_logging() {
77+
#ifdef USE_AZURE
78+
auto azureLogLevel =
79+
static_cast<Azure::Core::Diagnostics::Logger::Level>(config::azure_log_level);
80+
Azure::Core::Diagnostics::Logger::SetLevel(azureLogLevel);
81+
Azure::Core::Diagnostics::Logger::SetListener(
82+
[&](Azure::Core::Diagnostics::Logger::Level level, const std::string& message) {
83+
switch (level) {
84+
case Azure::Core::Diagnostics::Logger::Level::Verbose:
85+
LOG(INFO) << message;
86+
break;
87+
case Azure::Core::Diagnostics::Logger::Level::Informational:
88+
LOG(INFO) << message;
89+
break;
90+
case Azure::Core::Diagnostics::Logger::Level::Warning:
91+
LOG(WARNING) << message;
92+
break;
93+
case Azure::Core::Diagnostics::Logger::Level::Error:
94+
LOG(ERROR) << message;
95+
break;
96+
default:
97+
LOG(WARNING) << "Unknown level: " << static_cast<int>(level)
98+
<< ", message: " << message;
99+
break;
100+
}
101+
});
102+
#endif
103+
}
104+
75105
Result<std::shared_ptr<ObjStorageClient>> AzureClientProvider::get_or_create(
76106
const ObjClientConfig& config, const AzureClientConfig& azure) {
77107
if (auto cached = _cache.find(config)) {
@@ -98,7 +128,7 @@ Result<std::shared_ptr<ObjStorageClient>> AzureClientProvider::_build(
98128
options.Retry.StatusCodes.insert(Azure::Core::Http::HttpStatusCode::TooManyRequests);
99129
options.Retry.MaxRetries = config::max_s3_client_retry;
100130
options.PerRetryPolicies.emplace_back(std::make_unique<AzureRetryRecordPolicy>());
101-
auto ca_cert_file_path = S3ClientFactory::instance().ca_cert_file_path();
131+
auto ca_cert_file_path = obj_client_ca_cert_file_path();
102132
if (!ca_cert_file_path.empty()) {
103133
Azure::Core::Http::CurlTransportOptions curl_options;
104134
curl_options.CAInfo = ca_cert_file_path;

‎be/src/io/obj_storage/azure_client_provider.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,10 @@
2525

2626
namespace doris::io {
2727

28+
// Routes the Azure SDK log to the BE log at config::azure_log_level. ObjClientFactory calls it
29+
// once on construction, which ExecEnv triggers at BE startup. Does nothing with BUILD_AZURE off.
30+
void init_azure_sdk_logging();
31+
2832
// Builds, caches and reuses Azure Blob SDK clients. The header is SDK-free so that the
2933
// factory compiles with BUILD_AZURE off; _build reports NotSupported in that case.
3034
class AzureClientProvider {
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "io/obj_storage/obj_client_ca.h"
19+
20+
#include <mutex>
21+
#include <utility>
22+
23+
#include "common/config.h"
24+
#include "cpp/aws_common.h"
25+
#include "util/string_util.h"
26+
27+
namespace doris::io {
28+
namespace {
29+
30+
struct CaCertCache {
31+
std::mutex lock;
32+
std::string path;
33+
};
34+
35+
// A function-local static, so that callers during static initialization find it constructed.
36+
CaCertCache& ca_cert_cache() {
37+
static CaCertCache cache;
38+
return cache;
39+
}
40+
41+
} // namespace
42+
43+
std::string obj_client_ca_cert_file_path() {
44+
auto& cache = ca_cert_cache();
45+
std::lock_guard lock(cache.lock);
46+
if (cache.path.empty()) {
47+
cache.path = get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";"));
48+
}
49+
return cache.path;
50+
}
51+
52+
#ifdef BE_TEST
53+
std::string exchange_obj_client_ca_cert_file_path_for_test(std::string path) {
54+
auto& cache = ca_cert_cache();
55+
std::lock_guard lock(cache.lock);
56+
return std::exchange(cache.path, std::move(path));
57+
}
58+
#endif
59+
60+
} // namespace doris::io
Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#pragma once
19+
20+
#include <string>
21+
22+
namespace doris::io {
23+
24+
// The CA bundle that object storage clients of every backend, and the AWS credentials
25+
// providers, verify TLS with: the first valid path of config::ca_cert_file_paths. It is resolved
26+
// lazily and cached; while no path is valid the result is empty and the next call looks again.
27+
std::string obj_client_ca_cert_file_path();
28+
29+
#ifdef BE_TEST
30+
// Replaces the cached path and returns the previous one; an empty path makes the next
31+
// obj_client_ca_cert_file_path() resolve it again.
32+
std::string exchange_obj_client_ca_cert_file_path_for_test(std::string path);
33+
#endif
34+
35+
} // namespace doris::io

‎be/src/io/obj_storage/obj_client_factory.cpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,9 @@ namespace doris::io {
2626
ObjClientFactory::ObjClientFactory() {
2727
// Complete AWS SDK initialization before this singleton finishes construction. Function
2828
// local statics are destroyed in reverse order of construction, so the cached AWS
29-
// clients held by the providers are released before S3ClientFactory shuts the SDK down.
30-
S3ClientFactory::instance();
29+
// clients held by the providers are released before AwsSdkManager shuts the SDK down.
30+
AwsSdkManager::instance();
31+
init_azure_sdk_logging();
3132
}
3233

3334
ObjClientFactory& ObjClientFactory::instance() {

‎be/src/io/obj_storage/s3_client_provider.cpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
#include "cpp/obj_retry_strategy.h"
3333
#include "cpp/sync_point.h"
3434
#include "io/fs/gcs_signed_url_provider.h"
35+
#include "io/obj_storage/obj_client_ca.h"
3536
#include "io/obj_storage/obj_client_rate_limit.h"
3637
#include "util/s3_util.h"
3738

@@ -57,13 +58,13 @@ Result<std::shared_ptr<ObjStorageClient>> S3ClientProvider::_build(
5758
"s3_client_factory::create",
5859
std::make_shared<S3ObjStorageClient>(std::make_shared<Aws::S3::S3Client>(),
5960
ObjStorageEndpointInfo {}));
60-
Aws::Client::ClientConfiguration aws_config = S3ClientFactory::getClientConfiguration();
61+
Aws::Client::ClientConfiguration aws_config = AwsSdkManager::getClientConfiguration();
6162
if (s3.need_override_endpoint) {
6263
aws_config.endpointOverride = s3.endpoint;
6364
}
6465
aws_config.region = s3.region;
6566

66-
auto ca_cert_file_path = S3ClientFactory::instance().ca_cert_file_path();
67+
auto ca_cert_file_path = obj_client_ca_cert_file_path();
6768
if (!ca_cert_file_path.empty()) {
6869
aws_config.caFile = ca_cert_file_path;
6970
}
@@ -89,7 +90,7 @@ Result<std::shared_ptr<ObjStorageClient>> S3ClientProvider::_build(
8990
aws_config.retryStrategy = std::make_shared<S3CustomRetryStrategy>(
9091
config::max_s3_client_retry /*scaleFactor = 25*/, /*retry_slow_down=*/true);
9192

92-
auto credentials = S3ClientFactory::instance().create_aws_credentials_provider(s3.auth);
93+
auto credentials = AwsSdkManager::instance().create_aws_credentials_provider(s3.auth);
9394
if (!credentials) {
9495
return ResultError(Status::InvalidArgument("failed to create AWS credential provider: {}",
9596
credentials.error));

‎be/src/load/routine_load/data_consumer.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -674,7 +674,7 @@ Status KinesisDataConsumer::init(std::shared_ptr<StreamLoadContext> ctx) {
674674
}
675675

676676
Status KinesisDataConsumer::_create_kinesis_client(std::shared_ptr<StreamLoadContext> ctx) {
677-
// Reuse S3ClientFactory's credential provider logic
677+
// Reuse AwsSdkManager's credential provider logic
678678
// This supports all AWS authentication methods:
679679
// - Simple AK/SK
680680
// - IAM instance profile (EC2)
@@ -713,7 +713,7 @@ Status KinesisDataConsumer::_create_kinesis_client(std::shared_ptr<StreamLoadCon
713713
}
714714

715715
// Create AWS ClientConfiguration
716-
Aws::Client::ClientConfiguration aws_config = S3ClientFactory::getClientConfiguration();
716+
Aws::Client::ClientConfiguration aws_config = AwsSdkManager::getClientConfiguration();
717717
aws_config.region = _region;
718718

719719
if (!_endpoint.empty()) {
@@ -752,7 +752,7 @@ Status KinesisDataConsumer::_create_kinesis_client(std::shared_ptr<StreamLoadCon
752752
}
753753

754754
// Create credentials provider (reuses S3 infrastructure)
755-
auto credentials = S3ClientFactory::instance().create_aws_credentials_provider(auth);
755+
auto credentials = AwsSdkManager::instance().create_aws_credentials_provider(auth);
756756
if (!credentials) {
757757
return Status::InvalidArgument("Failed to create AWS credential provider: {}",
758758
credentials.error);

‎be/src/runtime/exec_env_init.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@
6767
#include "io/fs/hdfs_file_writer.h"
6868
#include "io/fs/local_file_reader.h"
6969
#include "io/fs/read_io_trace.h"
70+
#include "io/obj_storage/obj_client_factory.h"
7071
#include "load/channel/load_channel_mgr.h"
7172
#include "load/channel/load_stream_mgr.h"
7273
#include "load/group_commit/group_commit_mgr.h"
@@ -506,8 +507,9 @@ Status ExecEnv::_init(const std::vector<StorePath>& store_paths,
506507

507508
init_simdjson_parser();
508509

509-
// Make aws-sdk-cpp InitAPI and ShutdownAPI called in the same thread
510-
S3ClientFactory::instance();
510+
// Initialize the object storage SDKs: AWS InitAPI, so that ShutdownAPI runs in the same
511+
// thread, and the Azure SDK log.
512+
io::ObjClientFactory::instance();
511513
return Status::OK();
512514
}
513515

‎be/src/util/s3_rate_limiter_manager.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ enum class S3RateLimitRejectReason {
5959
};
6060

6161
// Owns the 4 process-wide token buckets (GET/PUT x QPS/bytes). Independent of
62-
// S3ClientFactory so that instantiating it never initializes the AWS SDK.
62+
// AwsSdkManager so that instantiating it never initializes the AWS SDK.
6363
class S3RateLimiterManager {
6464
public:
6565
static S3RateLimiterManager& instance();

‎be/src/util/s3_util.cpp‎

Lines changed: 9 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -20,79 +20,41 @@
2020
#include <aws/core/utils/logging/LogLevel.h>
2121
#include <aws/core/utils/logging/LogSystemInterface.h>
2222

23-
#ifdef USE_AZURE
24-
#include <azure/core/diagnostics/logger.hpp>
25-
#endif
2623
#include <memory>
2724
#include <utility>
2825

2926
#include "common/config.h"
3027
#include "common/logging.h"
3128
#include "cpp/aws_logger.h"
3229
#include "cpp/obj-client/auth/aws_credential_factory.h"
30+
#include "io/obj_storage/obj_client_ca.h"
3331
#include "io/obj_storage/obj_storage_config.h"
34-
#include "util/string_util.h"
3532

3633
namespace doris {
3734

38-
S3ClientFactory::S3ClientFactory() {
35+
AwsSdkManager::AwsSdkManager() {
3936
_aws_options = Aws::SDKOptions {};
4037
auto logLevel = static_cast<Aws::Utils::Logging::LogLevel>(config::aws_log_level);
4138
_aws_options.loggingOptions.logLevel = logLevel;
4239
_aws_options.loggingOptions.logger_create_fn = [logLevel] {
4340
return std::make_shared<DorisAWSLogger>(logLevel);
4441
};
4542
Aws::InitAPI(_aws_options);
46-
_get_ca_cert_file_path();
47-
48-
#ifdef USE_AZURE
49-
auto azureLogLevel =
50-
static_cast<Azure::Core::Diagnostics::Logger::Level>(config::azure_log_level);
51-
Azure::Core::Diagnostics::Logger::SetLevel(azureLogLevel);
52-
Azure::Core::Diagnostics::Logger::SetListener(
53-
[&](Azure::Core::Diagnostics::Logger::Level level, const std::string& message) {
54-
switch (level) {
55-
case Azure::Core::Diagnostics::Logger::Level::Verbose:
56-
LOG(INFO) << message;
57-
break;
58-
case Azure::Core::Diagnostics::Logger::Level::Informational:
59-
LOG(INFO) << message;
60-
break;
61-
case Azure::Core::Diagnostics::Logger::Level::Warning:
62-
LOG(WARNING) << message;
63-
break;
64-
case Azure::Core::Diagnostics::Logger::Level::Error:
65-
LOG(ERROR) << message;
66-
break;
67-
default:
68-
LOG(WARNING) << "Unknown level: " << static_cast<int>(level)
69-
<< ", message: " << message;
70-
break;
71-
}
72-
});
73-
#endif
43+
io::obj_client_ca_cert_file_path();
7444
}
7545

76-
S3ClientFactory::~S3ClientFactory() {
46+
AwsSdkManager::~AwsSdkManager() {
7747
Aws::ShutdownAPI(_aws_options);
7848
}
7949

80-
S3ClientFactory& S3ClientFactory::instance() {
81-
static S3ClientFactory ret;
50+
AwsSdkManager& AwsSdkManager::instance() {
51+
static AwsSdkManager ret;
8252
return ret;
8353
}
8454

85-
std::string S3ClientFactory::_get_ca_cert_file_path() {
86-
std::lock_guard lock(_ca_cert_lock);
87-
if (_ca_cert_file_path.empty()) {
88-
_ca_cert_file_path = get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";"));
89-
}
90-
return _ca_cert_file_path;
91-
}
92-
93-
AwsCredentialResult S3ClientFactory::create_aws_credentials_provider(const io::S3AuthConfig& auth) {
94-
auto sts_config = S3ClientFactory::getClientConfiguration();
95-
auto ca_cert_file_path = _get_ca_cert_file_path();
55+
AwsCredentialResult AwsSdkManager::create_aws_credentials_provider(const io::S3AuthConfig& auth) {
56+
auto sts_config = AwsSdkManager::getClientConfiguration();
57+
auto ca_cert_file_path = io::obj_client_ca_cert_file_path();
9658
if (!ca_cert_file_path.empty()) {
9759
sts_config.caFile = ca_cert_file_path;
9860
}

0 commit comments

Comments
 (0)