Skip to content

Commit d3b60b3

Browse files
chenjunweichenjunwei
authored andcommitted
[fix](be) Preserve the vault SSE flag in storage diagnostics
### What problem does this PR solve? Related PR: apache#69605 Problem Summary: Migrating the BE storage binding from S3Conf dropped the sse_enabled value carried by ObjectStoreInfoPB. The previous BE included that value in vault creation and error diagnostics. Carry it on ObjStorageConfig, keep the original false default and print it in the storage and flat-source diagnostics. Client construction, caching and the upload path continue to use their existing inputs. ### Release note BE vault diagnostics retain the sse_enabled value from storage metadata. ### Check List (For Author) - Test: Built FE and the ASAN BE with Azure enabled. Build hygiene, clang-format 16 and clang-tidy passed. The new test failed on the previous code for both missing log fields and passed after the fix. 242 selected ASAN_UT cases passed, with 34 external-storage cases skipped. - Behavior changed: No relative to master. The refactored BE reports the vault SSE value again. - Does this need documentation: No
1 parent 4af1d44 commit d3b60b3

4 files changed

Lines changed: 30 additions & 7 deletions

File tree

‎be/src/io/obj_storage/obj_storage_config.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,8 @@ std::string ObjClientConfig::to_string() const {
195195
}
196196

197197
std::string ObjStorageConfig::to_string() const {
198-
return fmt::format("(bucket={}, prefix={}, client={})", bucket, prefix, client.to_string());
198+
return fmt::format("(bucket={}, prefix={}, client={}, sse_enabled={})", bucket, prefix,
199+
client.to_string(), sse_enabled);
199200
}
200201

201202
Status validate_obj_client_config(const ObjClientConfig& config) {

‎be/src/io/obj_storage/obj_storage_config.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,8 @@ struct ObjStorageConfig {
134134
std::string bucket;
135135
std::string prefix;
136136
ObjClientConfig client;
137+
// Vault metadata reported in BE diagnostics.
138+
bool sse_enabled = false;
137139

138140
// Credentials are masked.
139141
std::string to_string() const;

‎be/src/io/obj_storage/obj_storage_config_resolver.cpp‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -85,17 +85,17 @@ bool flat_to_int(std::string_view str, int& res) {
8585
return ec == std::errc {};
8686
}
8787

88-
std::string flat_fields_to_string(const FlatClientFields& fields) {
88+
std::string flat_fields_to_string(const FlatClientFields& fields, bool sse_enabled) {
8989
return fmt::format(
9090
"(ak={}, token={}, endpoint={}, region={}, bucket={}, max_connections={}, "
9191
"request_timeout_ms={}, connect_timeout_ms={}, use_virtual_addressing={}, "
9292
"cred_provider_type={},role_arn={}, external_id={}, credential_type={}, "
93-
"is_internal_bucket={}",
93+
"is_internal_bucket={}, sse_enabled={})",
9494
hide_access_key(fields.ak), fields.token.empty() ? "" : "******", fields.endpoint,
9595
fields.region, fields.bucket, fields.max_connections, fields.request_timeout_ms,
9696
fields.connect_timeout_ms, fields.use_virtual_addressing,
9797
static_cast<int>(fields.cred_provider_type), fields.role_arn, fields.external_id,
98-
credential_type_name(fields.credential), fields.is_internal_bucket);
98+
credential_type_name(fields.credential), fields.is_internal_bucket, sse_enabled);
9999
}
100100

101101
// Flat FE properties require both fields even on Azure. When both are missing, report
@@ -332,7 +332,7 @@ Result<ObjStorageConfig> ObjStorageConfigResolver::from_object_store_info(
332332
default:
333333
__builtin_unreachable();
334334
LOG_FATAL("unknown provider type {}, info {}", info.provider(),
335-
flat_fields_to_string(fields));
335+
flat_fields_to_string(fields, info.sse_enabled()));
336336
}
337337
auto client = flat_client_config(fields);
338338
if (!client.has_value()) {
@@ -342,6 +342,7 @@ Result<ObjStorageConfig> ObjStorageConfigResolver::from_object_store_info(
342342
.bucket = info.bucket(),
343343
.prefix = info.prefix(),
344344
.client = std::move(client).value(),
345+
.sse_enabled = info.sse_enabled(),
345346
};
346347
}
347348

@@ -374,7 +375,7 @@ Result<ObjStorageConfig> ObjStorageConfigResolver::from_storage_param(
374375
switch (param.provider) {
375376
case TObjStorageType::UNKNOWN:
376377
LOG_INFO("Receive one legal storage resource, set provider type to aws, param detail {}",
377-
flat_fields_to_string(fields));
378+
flat_fields_to_string(fields, /*sse_enabled=*/false));
378379
fields.provider = ObjStorageProvider::AWS;
379380
break;
380381
case TObjStorageType::AWS:
@@ -403,7 +404,7 @@ Result<ObjStorageConfig> ObjStorageConfigResolver::from_storage_param(
403404
break;
404405
default:
405406
LOG_FATAL("unknown provider type {}, info {}", param.provider,
406-
flat_fields_to_string(fields));
407+
flat_fields_to_string(fields, /*sse_enabled=*/false));
407408
__builtin_unreachable();
408409
}
409410
auto client = flat_client_config(fields);

‎be/test/io/obj_storage/obj_client_factory_test.cpp‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -838,4 +838,23 @@ TEST_F(ObjClientFactoryTest, ResolvesPropertiesAndVaults) {
838838
EXPECT_EQ(vault_auth.provider_type, CredProviderType::InstanceProfile);
839839
}
840840

841+
TEST_F(ObjClientFactoryTest, VaultSseSettingIsLoggedAndReusesTheClient) {
842+
cloud::ObjectStoreInfoPB vault;
843+
vault.set_provider(cloud::ObjectStoreInfoPB_Provider_S3);
844+
vault.set_endpoint(unique_endpoint("vault-sse"));
845+
vault.set_region("us-east-1");
846+
vault.set_bucket("bucket");
847+
848+
auto storage = TEST_TRY(ObjStorageConfigResolver::from_object_store_info(vault));
849+
EXPECT_NE(storage.to_string().find("sse_enabled=false"), std::string::npos);
850+
auto& factory = ObjClientFactory::instance();
851+
auto client = TEST_TRY(factory.get_or_create(storage.client));
852+
853+
vault.set_sse_enabled(true);
854+
auto updated = TEST_TRY(ObjStorageConfigResolver::from_object_store_info(vault));
855+
EXPECT_NE(updated.to_string().find("sse_enabled=true"), std::string::npos);
856+
auto cached = TEST_TRY(factory.get_or_create(updated.client));
857+
EXPECT_EQ(client, cached);
858+
}
859+
841860
} // namespace doris::io

0 commit comments

Comments
 (0)