Skip to content

Commit e0caadd

Browse files
committed
[refactor](be) Let the last overlapping holder reset win
### What problem does this PR solve? Related PR: apache#69605 Problem Summary: ObjClientHolder published a new state only when no other reset had published since the caller took its snapshot, and otherwise returned Aborted. Before the refactor the holder built the client outside the lock and the last reset to publish won, with both calls succeeding. The generation check did not add correctness: the merge keeps only the storage target, which no reset changes, so the published configuration is the same whichever state it was merged with, and the holder cannot tell which caller carries the newer configuration either way. Its only effect was a different arbitrary winner and a failed refresh for the other caller. Restore the previous behavior: _publish no longer takes the expected state, and it assigns the next generation under the lock. The check for an invalidated holder stays, so a client that was being built during invalidate() still cannot bring the holder back. A reset cannot overlap a rebind: resource updates run on a single worker thread, and vault syncs only reset or invalidate. ### Release note None ### Check List (For Author) - Test: Not compiled and not run; this machine has no BE build. The unit test for overlapping resets was rewritten to expect both resets to succeed and the last one to stay, and needs a CI run. Build hygiene passed. Formatting of the changed lines was checked with clang-format 20 only, because clang-format 16 is not installed here. - Behavior changed: No relative to master. Overlapping resets both succeed and the last one wins again. - Does this need documentation: No
1 parent 1f587a4 commit e0caadd

3 files changed

Lines changed: 33 additions & 47 deletions

File tree

‎be/src/io/obj_storage/obj_client_holder.cpp‎

Lines changed: 14 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ ObjClientHolder::~ObjClientHolder() = default;
3838
Status ObjClientHolder::init() {
3939
auto current = state();
4040
auto client = DORIS_TRY(ObjClientFactory::instance().get_or_create(current->config));
41-
return _publish(current, current->config, std::move(client));
41+
return _publish(current->config, std::move(client));
4242
}
4343

4444
Status ObjClientHolder::reset(const ObjClientConfig& update) {
@@ -57,7 +57,7 @@ Status ObjClientHolder::reset(const ObjClientConfig& update) {
5757

5858
LOG(WARNING) << "reset object storage client with new config: " << update.to_string();
5959

60-
return _publish(current, std::move(merged), std::move(client));
60+
return _publish(std::move(merged), std::move(client));
6161
}
6262

6363
Status ObjClientHolder::rebind(const ObjClientConfig& config) {
@@ -66,7 +66,7 @@ Status ObjClientHolder::rebind(const ObjClientConfig& config) {
6666
return *current->invalid_reason;
6767
}
6868
auto client = DORIS_TRY(ObjClientFactory::instance().get_or_create(config));
69-
return _publish(current, config, std::move(client));
69+
return _publish(config, std::move(client));
7070
}
7171

7272
void ObjClientHolder::invalidate(Status reason) {
@@ -86,33 +86,22 @@ void ObjClientHolder::invalidate(Status reason) {
8686
_state.swap(next);
8787
}
8888

89-
Status ObjClientHolder::_publish(const std::shared_ptr<const ObjClientState>& expected,
90-
ObjClientConfig config, std::shared_ptr<ObjStorageClient> client) {
89+
Status ObjClientHolder::_publish(ObjClientConfig config, std::shared_ptr<ObjStorageClient> client) {
9190
// Declared before the lock: after the swap below it holds the replaced state, and
9291
// possibly the last reference to its client, which is then released after unlocking.
93-
std::shared_ptr<const ObjClientState> next = std::make_shared<ObjClientState>(ObjClientState {
92+
std::shared_ptr<const ObjClientState> next;
93+
std::lock_guard lock(_mtx);
94+
// A publish that overlapped invalidate() reports why the holder is out of service, so a
95+
// client that was already being built cannot bring it back.
96+
if (_state->invalid_reason.has_value()) {
97+
return *_state->invalid_reason;
98+
}
99+
next = std::make_shared<ObjClientState>(ObjClientState {
94100
.config = std::move(config),
95101
.client = std::move(client),
96-
.generation = expected->generation + 1,
102+
.generation = _state->generation + 1,
97103
});
98-
{
99-
std::lock_guard lock(_mtx);
100-
// Checked before the generation: a publish that overlapped invalidate() reports why the
101-
// holder is out of service rather than a conflict that the caller could retry.
102-
if (_state->invalid_reason.has_value()) {
103-
return *_state->invalid_reason;
104-
}
105-
// Two resets that both start from generation N merge their config onto N. Publishing
106-
// the second one on top of N + 1 would silently drop the first update, so it is
107-
// rejected and the caller retries from the new state.
108-
if (_state->generation != expected->generation) {
109-
return Status::Aborted(
110-
"object storage client was updated concurrently, expected generation {}, "
111-
"current {}",
112-
expected->generation, _state->generation);
113-
}
114-
_state.swap(next);
115-
}
104+
_state.swap(next);
116105
return Status::OK();
117106
}
118107

‎be/src/io/obj_storage/obj_client_holder.h‎

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -64,21 +64,18 @@ class ObjClientHolder {
6464
// the client if the result differs. This method is threadsafe. Resource and Storage Vault
6565
// refreshes are the callers.
6666
//
67-
// The new client is built outside the lock and published only if no other reset or
68-
// init published in between. Otherwise the update is rejected with Aborted rather than
69-
// applied on top of a state it was not merged with, so overlapping resets cannot both
70-
// win; the caller keeps the current state and retries with its next refresh.
71-
//
72-
// This detects overlap, not freshness: the holder cannot tell which caller fetched the
73-
// newer configuration. A reset that starts after another one published still wins with
74-
// whatever it carries, and of two overlapping resets the first to publish wins. Either
75-
// way an older configuration may stay until the next refresh replaces it.
67+
// The new client is built outside the lock. Of two overlapping resets the last to publish
68+
// wins, as before the state became immutable. The merge keeps only the storage target,
69+
// which no reset changes, so the winner carries its whole update whichever state it was
70+
// merged with. For example, reset(sk1) and reset(sk2) that overlap leave sk1 or sk2, never
71+
// a mix. The holder cannot tell which caller fetched the newer configuration, so an older
72+
// one may stay until the next refresh replaces it.
7673
Status reset(const ObjClientConfig& update);
7774

7875
// Replaces the state with `config` as it is, without merging, which allows another backend.
7976
// Readers keep the holder they resolved once, so moving the holder to another backend while
8077
// the bucket and prefix stay reaches all of them (S3FileSystem::refresh_resource).
81-
// The client is built and published like in reset(), with the same overlap check.
78+
// The client is built and published like in reset().
8279
Status rebind(const ObjClientConfig& config);
8380

8481
// Takes this holder out of service for the life of the process, because what it points at is
@@ -122,9 +119,8 @@ class ObjClientHolder {
122119
#endif
123120

124121
private:
125-
// Replaces the state with (config, client) if `expected` is still current.
126-
Status _publish(const std::shared_ptr<const ObjClientState>& expected, ObjClientConfig config,
127-
std::shared_ptr<ObjStorageClient> client);
122+
// Replaces the state with (config, client), unless the holder was invalidated.
123+
Status _publish(ObjClientConfig config, std::shared_ptr<ObjStorageClient> client);
128124

129125
mutable std::shared_mutex _mtx;
130126
std::shared_ptr<const ObjClientState> _state;

‎be/test/io/obj_storage/obj_client_factory_test.cpp‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -451,8 +451,8 @@ TEST_F(ObjClientFactoryTest, HolderPublishesConfigAndClientTogether) {
451451
EXPECT_EQ(holder.state(), current);
452452
}
453453

454-
TEST_F(ObjClientFactoryTest, HolderRejectsResetMergedOntoStaleState) {
455-
auto param = make_s3_param(unique_endpoint("holder-conflict"));
454+
TEST_F(ObjClientFactoryTest, OverlappingResetsLetTheLastPublishWin) {
455+
auto param = make_s3_param(unique_endpoint("holder-overlap"));
456456
ObjClientHolder holder(resolve(param));
457457
ASSERT_TRUE(holder.init().ok());
458458

@@ -461,18 +461,19 @@ TEST_F(ObjClientFactoryTest, HolderRejectsResetMergedOntoStaleState) {
461461
auto inner = param;
462462
inner.__set_sk("inner-sk");
463463
// While the outer reset builds its client outside the lock, another reset publishes
464-
// generation 2. The outer reset was merged onto generation 1, so publishing it would
465-
// overwrite the inner update with a config that never saw it.
464+
// generation 2. The outer reset then publishes generation 3 on top of it: both succeed and
465+
// the last one stays, with its whole update.
466466
Status inner_status;
467467
_on_s3_build = [&] { inner_status = holder.reset(resolve(inner)); };
468468

469469
auto outer_status = holder.reset(resolve(outer));
470470

471471
ASSERT_TRUE(inner_status.ok()) << inner_status;
472-
EXPECT_TRUE(outer_status.is<ErrorCode::ABORTED>()) << outer_status;
472+
ASSERT_TRUE(outer_status.ok()) << outer_status;
473473
auto current = holder.state();
474-
EXPECT_EQ(current->generation, 2U);
475-
EXPECT_EQ(secret_key_of(current->config), "inner-sk");
474+
EXPECT_EQ(current->generation, 3U);
475+
EXPECT_EQ(secret_key_of(current->config), "outer-sk");
476+
EXPECT_EQ(holder.get(), current->client);
476477
}
477478

478479
TEST_F(ObjClientFactoryTest, ResetMergeKeepsTheStorageTarget) {
@@ -561,8 +562,8 @@ TEST_F(ObjClientFactoryTest, InvalidateWinsOverAResetBuildingItsClient) {
561562
_on_s3_build = [&] { holder.invalidate(Status::InvalidArgument<false>("backend changed")); };
562563
auto st = holder.reset(resolve(rotated));
563564

564-
// The publish reports the reason rather than Aborted, which callers treat as retryable.
565-
EXPECT_FALSE(st.is<ErrorCode::ABORTED>()) << st;
565+
// The publish reports the reason instead of bringing the client back.
566+
EXPECT_FALSE(st.ok()) << st;
566567
EXPECT_NE(st.to_string().find("backend changed"), std::string::npos) << st;
567568
EXPECT_EQ(holder.get(), nullptr);
568569
}

0 commit comments

Comments
 (0)