Skip to content

Commit d34f06b

Browse files
[fix](be): fix BE access to aws under eks pod identity (#67076)
### What problem does this PR solve? Issue Number: close #66554 **Problem Summary** Fixes the issue above by forwarding `AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE` alongside the inline token. Given a file path, the provider will re-read it before every fetch, so credentials survive token rotation. ECS, which sets only the inline variable `AWS_CONTAINER_AUTHORIZATION_TOKEN`, is unaffected. **Why the aws-sdp-cpp bump** The `GeneralHTTPCredentialsProvider` does not exist before 1.11.221. The only container provider available at the currently pinned version is `TaskRoleCredentialsProvider`, which takes the token as a string and has no notion of a token file. 1.11.221 is the first version whose `GeneralHTTPCredentialsProvider` re-reads the token file.
1 parent 38339c5 commit d34f06b

12 files changed

Lines changed: 657 additions & 57 deletions

‎be/src/runtime/aws_msk_iam_auth.cpp‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
#include <aws/core/auth/AWSCredentialsProvider.h>
2222
#include <aws/core/auth/AWSCredentialsProviderChain.h>
2323
#include <aws/core/auth/STSCredentialsProvider.h>
24-
#include <aws/core/platform/Environment.h>
2524
#include <aws/identity-management/auth/STSAssumeRoleCredentialsProvider.h>
2625
#include <aws/sts/STSClient.h>
2726
#include <aws/sts/model/AssumeRoleRequest.h>
@@ -34,6 +33,7 @@
3433
#include <sstream>
3534

3635
#include "common/logging.h"
36+
#include "cpp/aws_common.h"
3737

3838
namespace doris {
3939

@@ -51,8 +51,7 @@ std::shared_ptr<Aws::Auth::AWSCredentialsProvider> AwsMskIamAuth::_create_provid
5151
} else if (provider_upper == "INSTANCE_PROFILE" || provider_upper == "INSTANCEPROFILE") {
5252
return std::make_shared<Aws::Auth::InstanceProfileCredentialsProvider>();
5353
} else if (provider_upper == "CONTAINER" || provider_upper == "ECS") {
54-
return std::make_shared<Aws::Auth::TaskRoleCredentialsProvider>(
55-
Aws::Environment::GetEnv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI").c_str());
54+
return create_container_credentials_provider();
5655
} else if (provider_upper == "SYSTEM_PROPERTIES" || provider_upper == "SYSTEMPROPERTIES") {
5756
return std::make_shared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>();
5857
} else if (provider_upper == "WEB_IDENTITY" || provider_upper == "WEBIDENTITY" ||

‎be/test/io/s3_client_factory_test.cpp‎

Lines changed: 197 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,19 @@
1616
// under the License.
1717

1818
#include <aws/core/auth/AWSCredentialsProviderChain.h>
19+
#include <aws/core/auth/GeneralHTTPCredentialsProvider.h>
1920
#include <aws/core/auth/STSCredentialsProvider.h>
2021
#include <aws/core/client/ClientConfiguration.h>
2122
#include <aws/identity-management/auth/STSAssumeRoleCredentialsProvider.h>
2223
#include <aws/s3/model/HeadObjectResult.h>
2324
#include <gtest/gtest.h>
2425
#include <unistd.h>
2526

27+
#include <cstdio>
2628
#include <cstdlib>
2729
#include <filesystem>
2830
#include <fstream>
31+
#include <memory>
2932
#include <string>
3033
#include <utility>
3134
#include <vector>
@@ -34,9 +37,11 @@
3437
#include "common/config.h"
3538
#include "cpp/aws_common.h"
3639
#include "cpp/custom_aws_credentials_provider_chain.h"
40+
#include "cpp/obj-client/auth/aws_credential_factory.h"
3741
#include "cpp/obj-client/s3_obj_storage_client.h"
3842
#include "cpp/sync_point.h"
3943
#include "io/fs/s3_file_system.h"
44+
#include "testutil/container_credentials_endpoint.h"
4045
#include "util/s3_rate_limiter_manager.h"
4146
#include "util/s3_uri.h"
4247
#include "util/s3_util.h"
@@ -573,16 +578,15 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProviderV2ProviderTypeWithoutRoleArn)
573578
provider),
574579
nullptr);
575580

576-
const char* old_container_uri = std::getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI");
577-
if (old_container_uri == nullptr) {
578-
setenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "/v2/credentials/mock", 1);
579-
}
580-
S3ClientConf container_conf;
581-
container_conf.cred_provider_type = CredProviderType::Container;
582-
provider = factory.create_aws_credentials_provider(container_conf).provider;
583-
ASSERT_NE(std::dynamic_pointer_cast<Aws::Auth::TaskRoleCredentialsProvider>(provider), nullptr);
584-
if (old_container_uri == nullptr) {
585-
unsetenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI");
581+
{
582+
// The environment is pinned rather than merely topped up, so that this case asserts the
583+
// same thing whether or not the machine running it happens to be inside a container.
584+
ContainerCredentialsEnvGuard env;
585+
env.set_ecs_task_role("/v2/credentials/mock");
586+
S3ClientConf container_conf;
587+
container_conf.cred_provider_type = CredProviderType::Container;
588+
provider = factory.create_aws_credentials_provider(container_conf).provider;
589+
ASSERT_NE(as_valid_http_provider(provider), nullptr);
586590
}
587591

588592
S3ClientConf instance_profile_conf;
@@ -689,4 +693,187 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProviderV1PartialCredentialsUseDefault
689693
config::aws_credentials_provider_version = "v2";
690694
}
691695

696+
namespace {
697+
698+
// The provider is looked up inside the chain rather than constructed directly,
699+
// because what is under test is how CustomAwsCredentialsProviderChain wires the
700+
// environment.
701+
Aws::Auth::GeneralHTTPCredentialsProvider* find_http_provider(
702+
const CustomAwsCredentialsProviderChain& chain) {
703+
for (const auto& provider : chain.GetProviders()) {
704+
auto* http_provider =
705+
dynamic_cast<Aws::Auth::GeneralHTTPCredentialsProvider*>(provider.get());
706+
if (http_provider != nullptr) {
707+
return http_provider;
708+
}
709+
}
710+
return nullptr;
711+
}
712+
713+
} // namespace
714+
715+
// EKS Pod Identity supplies the credential-endpoint token as a file that the
716+
// kubelet rotates in place, and never as a plain environment variable. The chain
717+
// must therefore hand the provider the token path, so that every refresh picks
718+
// up the current contents. Passing the value read at construction time works
719+
// until the first rotation and then fails with AccessDenied.
720+
TEST_F(S3ClientFactoryTest, CustomChainReadsRotatedTokenFileForPodIdentity) {
721+
(void)S3ClientFactory::instance();
722+
723+
ContainerCredentialsEndpoint endpoint;
724+
ASSERT_TRUE(endpoint.start());
725+
726+
ContainerCredentialsEnvGuard env;
727+
const std::string token_path = env.token_file_path("custom_chain");
728+
env.write_token_file(token_path, "token-one");
729+
env.set_pod_identity(endpoint.url(), token_path);
730+
731+
CustomAwsCredentialsProviderChain chain;
732+
auto* provider = find_http_provider(chain);
733+
ASSERT_NE(provider, nullptr) << "no GeneralHTTPCredentialsProvider was added to the chain";
734+
735+
// Each GetAWSCredentials() drives one HTTP GET to the endpoint: the provider re-reads the token
736+
// file, sends it as the Authorization header, and parses the credentials from the reply. The
737+
// handler records the header it saw, so auth_headers() reports what actually went over the wire.
738+
const auto first_credentials = provider->GetAWSCredentials();
739+
EXPECT_EQ(first_credentials.GetAWSAccessKeyId(), "AKIDTEST");
740+
EXPECT_EQ(first_credentials.GetSessionToken(), "SESSIONTEST");
741+
742+
// Rotate the file the way the kubelet does. The second call refetches rather than serving its
743+
// cache only because the handler reports an already-expired Expiration.
744+
env.write_token_file(token_path, "token-two");
745+
provider->GetAWSCredentials();
746+
747+
// GE rather than EQ: the credentials client retries and the handler records every attempt, so an
748+
// exact count would make a retry look like a bug. front/back keep the assertion at full strength
749+
// because every request before the rewrite carries token-one and every one after carries token-two.
750+
const auto auth_headers = endpoint.auth_headers();
751+
ASSERT_GE(auth_headers.size(), 2u);
752+
EXPECT_EQ(auth_headers.front(), "token-one");
753+
EXPECT_EQ(auth_headers.back(), "token-two");
754+
}
755+
756+
// The same wiring has to hold when CONTAINER is asked for by name rather than reached through the
757+
// default chain, which is the whole point of a public provider mode. Reading only
758+
// AWS_CONTAINER_CREDENTIALS_RELATIVE_URI leaves this provider with an empty URI, no HTTP client and
759+
// no credentials on every standard EKS deployment.
760+
TEST_F(S3ClientFactoryTest, ContainerProviderTypeReadsRotatedTokenFileForPodIdentity) {
761+
S3ClientFactory& factory = S3ClientFactory::instance();
762+
config::aws_credentials_provider_version = "v2";
763+
764+
ContainerCredentialsEndpoint endpoint;
765+
ASSERT_TRUE(endpoint.start());
766+
767+
ContainerCredentialsEnvGuard env;
768+
const std::string token_path = env.token_file_path("container_type");
769+
env.write_token_file(token_path, "token-one");
770+
env.set_pod_identity(endpoint.url(), token_path);
771+
772+
S3ClientConf conf;
773+
conf.cred_provider_type = CredProviderType::Container;
774+
auto provider = as_valid_http_provider(factory.create_aws_credentials_provider(conf).provider);
775+
ASSERT_NE(provider, nullptr) << "CONTAINER did not yield a usable container credentials "
776+
"provider for AWS_CONTAINER_CREDENTIALS_FULL_URI";
777+
778+
const auto first_credentials = provider->GetAWSCredentials();
779+
EXPECT_EQ(first_credentials.GetAWSAccessKeyId(), "AKIDTEST");
780+
EXPECT_EQ(first_credentials.GetSessionToken(), "SESSIONTEST");
781+
782+
env.write_token_file(token_path, "token-two");
783+
provider->GetAWSCredentials();
784+
785+
const auto auth_headers = endpoint.auth_headers();
786+
ASSERT_GE(auth_headers.size(), 2u);
787+
EXPECT_EQ(auth_headers.front(), "token-one");
788+
EXPECT_EQ(auth_headers.back(), "token-two");
789+
}
790+
791+
TEST_F(S3ClientFactoryTest, ContainerProviderTypeForwardsInlineAuthorizationToken) {
792+
S3ClientFactory& factory = S3ClientFactory::instance();
793+
config::aws_credentials_provider_version = "v2";
794+
795+
ContainerCredentialsEndpoint endpoint;
796+
ASSERT_TRUE(endpoint.start());
797+
798+
ContainerCredentialsEnvGuard env;
799+
env.set_inline_token_endpoint(endpoint.url(), "inline-token");
800+
801+
S3ClientConf conf;
802+
conf.cred_provider_type = CredProviderType::Container;
803+
auto provider = as_valid_http_provider(factory.create_aws_credentials_provider(conf).provider);
804+
ASSERT_NE(provider, nullptr) << "CONTAINER did not yield a usable container credentials "
805+
"provider for an inline authorization token";
806+
807+
EXPECT_EQ(provider->GetAWSCredentials().GetAWSAccessKeyId(), "AKIDTEST");
808+
809+
const auto auth_headers = endpoint.auth_headers();
810+
ASSERT_GE(auth_headers.size(), 1u);
811+
EXPECT_EQ(auth_headers.front(), "inline-token");
812+
}
813+
814+
TEST_F(S3ClientFactoryTest, ContainerProviderTypeStillHonoursEcsRelativeUri) {
815+
S3ClientFactory& factory = S3ClientFactory::instance();
816+
config::aws_credentials_provider_version = "v2";
817+
818+
ContainerCredentialsEnvGuard env;
819+
env.set_ecs_task_role("/v2/credentials/mock");
820+
821+
S3ClientConf conf;
822+
conf.cred_provider_type = CredProviderType::Container;
823+
EXPECT_NE(as_valid_http_provider(factory.create_aws_credentials_provider(conf).provider),
824+
nullptr)
825+
<< "CONTAINER did not yield a usable container credentials provider for "
826+
"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI";
827+
}
828+
829+
// With neither URI exported there is nothing to talk to, and the provider says so. This pins the
830+
// discriminator the two tests above rely on: IsValid() really does distinguish a wired provider
831+
// from an inert one, so their assertions cannot pass vacuously.
832+
TEST_F(S3ClientFactoryTest, ContainerProviderTypeIsUnusableWithoutAnyUri) {
833+
S3ClientFactory& factory = S3ClientFactory::instance();
834+
config::aws_credentials_provider_version = "v2";
835+
836+
// Constructing the guard is the whole setup: it clears all four AWS_CONTAINER_* variables, so
837+
// this runs as if on a host that is not a container at all.
838+
ContainerCredentialsEnvGuard env;
839+
840+
S3ClientConf conf;
841+
conf.cred_provider_type = CredProviderType::Container;
842+
auto provider = factory.create_aws_credentials_provider(conf).provider;
843+
ASSERT_NE(std::dynamic_pointer_cast<Aws::Auth::GeneralHTTPCredentialsProvider>(provider),
844+
nullptr);
845+
EXPECT_EQ(as_valid_http_provider(provider), nullptr);
846+
}
847+
848+
// With a role_arn the CONTAINER provider is not handed to the S3 client directly: it becomes the
849+
// base provider the STS client authenticates its AssumeRole call with. AwsCredentialFactory is
850+
// asked for that base provider here - a v2 request with no role_arn returns it unwrapped - so the
851+
// assertion is about the base itself rather than the STS wrapper around it.
852+
TEST_F(S3ClientFactoryTest, ContainerProviderTypeIsUsableAsStsBaseProvider) {
853+
(void)S3ClientFactory::instance();
854+
855+
ContainerCredentialsEndpoint endpoint;
856+
ASSERT_TRUE(endpoint.start());
857+
858+
ContainerCredentialsEnvGuard env;
859+
const std::string token_path = env.token_file_path("sts_base");
860+
env.write_token_file(token_path, "token-one");
861+
env.set_pod_identity(endpoint.url(), token_path);
862+
863+
AwsCredentialOptions options;
864+
options.version = AwsCredentialProviderVersion::V2;
865+
options.provider_type = CredProviderType::Container;
866+
// role_arn stays empty on purpose: that is what makes create() hand back the base provider
867+
// itself instead of the STS wrapper built on top of it.
868+
auto base_provider = as_valid_http_provider(AwsCredentialFactory::create(options).provider);
869+
ASSERT_NE(base_provider, nullptr)
870+
<< "CONTAINER did not yield a usable STS base credentials provider";
871+
872+
EXPECT_EQ(base_provider->GetAWSCredentials().GetAWSAccessKeyId(), "AKIDTEST");
873+
const auto auth_headers = endpoint.auth_headers();
874+
ASSERT_FALSE(auth_headers.empty())
875+
<< "the STS base provider never contacted the credentials endpoint";
876+
EXPECT_EQ(auth_headers.back(), "token-one");
877+
}
878+
692879
} // namespace doris

‎be/test/runtime/aws_msk_iam_auth_test.cpp‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,16 @@
1717

1818
#include "runtime/aws_msk_iam_auth.h"
1919

20+
#include <aws/core/auth/AWSCredentials.h>
2021
#include <gtest/gtest.h>
2122

2223
#include <memory>
2324
#include <string>
2425
#include <unordered_map>
2526

2627
#include "common/status.h"
28+
#include "testutil/container_credentials_endpoint.h"
29+
#include "util/s3_util.h"
2730

2831
namespace doris {
2932

@@ -34,6 +37,14 @@ class AwsMskIamAuthTest : public testing::Test {
3437
config.region = "us-east-1";
3538
}
3639

40+
// Nothing in this binary initialises the AWS SDK on its own - neither BE-UT's main() nor this
41+
// file - and without Aws::InitAPI there is no HTTP client factory, so a credentials provider
42+
// quietly returns nothing instead of making a request. Constructing the S3 client factory is
43+
// the cheapest in-tree way to get InitAPI called, exactly once per process, and it is the same
44+
// trick the S3 client factory tests use. Only the tests that drive a real fetch need it, so it
45+
// stays out of SetUp() where it would change what the older tests in this file do.
46+
static void ensure_aws_sdk_initialized() { (void)S3ClientFactory::instance(); }
47+
3748
AwsMskIamAuth::Config config;
3849
};
3950

@@ -215,6 +226,49 @@ TEST_F(AwsMskIamAuthTest, TestOAuthCallbackCreationWithExternalIdWithoutRoleArn)
215226
ASSERT_EQ(callback, nullptr);
216227
}
217228

229+
TEST_F(AwsMskIamAuthTest, ContainerProviderReadsTokenFileForPodIdentity) {
230+
ensure_aws_sdk_initialized();
231+
232+
ContainerCredentialsEndpoint endpoint;
233+
ASSERT_TRUE(endpoint.start());
234+
235+
ContainerCredentialsEnvGuard env;
236+
const std::string token_path = env.token_file_path("msk_container");
237+
env.write_token_file(token_path, "token-one");
238+
env.set_pod_identity(endpoint.url(), token_path);
239+
240+
config.credentials_provider = "CONTAINER";
241+
AwsMskIamAuth auth(config);
242+
243+
Aws::Auth::AWSCredentials credentials;
244+
ASSERT_TRUE(auth.get_credentials(&credentials).ok());
245+
EXPECT_EQ(credentials.GetAWSAccessKeyId(), "AKIDTEST");
246+
EXPECT_EQ(credentials.GetSessionToken(), "SESSIONTEST");
247+
248+
const auto auth_headers = endpoint.auth_headers();
249+
ASSERT_FALSE(auth_headers.empty());
250+
EXPECT_EQ(auth_headers.back(), "token-one");
251+
}
252+
253+
TEST_F(AwsMskIamAuthTest, EcsProviderAliasReachesContainerCredentialsEndpoint) {
254+
ensure_aws_sdk_initialized();
255+
256+
ContainerCredentialsEndpoint endpoint;
257+
ASSERT_TRUE(endpoint.start());
258+
259+
ContainerCredentialsEnvGuard env;
260+
const std::string token_path = env.token_file_path("msk_ecs_alias");
261+
env.write_token_file(token_path, "token-one");
262+
env.set_pod_identity(endpoint.url(), token_path);
263+
264+
config.credentials_provider = "ECS";
265+
AwsMskIamAuth auth(config);
266+
267+
Aws::Auth::AWSCredentials credentials;
268+
ASSERT_TRUE(auth.get_credentials(&credentials).ok());
269+
EXPECT_EQ(credentials.GetAWSAccessKeyId(), "AKIDTEST");
270+
}
271+
218272
// Integration test - only runs if AWS credentials are available
219273
TEST_F(AwsMskIamAuthTest, DISABLED_IntegrationTestWithRealCredentials) {
220274
// This test is disabled by default

0 commit comments

Comments
 (0)