Skip to content

Commit 2f8228c

Browse files
Recover from concurrent OAuth token cache writes (databricks#1793)
## Summary Recovers U2M OAuth refreshes when another process wins a concurrent token cache write. The process whose write fails briefly reloads the cache and returns the fresh token stored by the winner. ## Why Multiple CLI or SDK processes can read the same cached OAuth token, refresh it concurrently, and then race to replace the cache entry. On macOS, concurrent Keychain writes can cause one process to fail with `exit status 45`, even though another process successfully stored a valid token. This addresses [databricks/cli#6051](databricks/cli#6051) without adding cross-platform process locks. It does not prevent concurrent refreshes; it prevents a losing cache write from becoming an authentication failure when a fresh winner is available. ## What changed ### Interface changes None. ### Behavioral changes - After a refreshed token fails to be stored, `PersistentAuth` reloads the cache immediately and then after 25, 50, 100, and 200 milliseconds. - A changed, valid cached token is returned when its expiry is within one minute of the token obtained by the current process. - The original cache update error is preserved when no suitable token appears within the bounded 375-millisecond recovery window. ### Internal changes - Added freshness comparison and cache update recovery helpers. - Added tests covering delayed winner discovery, changed-token detection, validity, and expiry tolerance. - Added a bug-fix changelog entry. ## How is this tested? - `make test` (race-enabled full unit test suite) - `make lint` - `go test -count=1 ./credentials/u2m/...` The real macOS Keychain race has not yet been stress-tested on macOS. The unit test models the observed sequence: the cache write fails, the first reload still sees the original token, and a later reload sees the fresh token stored by another process. --------- Signed-off-by: Mihai Mitrea <[email protected]>
1 parent 92ab395 commit 2f8228c

3 files changed

Lines changed: 196 additions & 0 deletions

File tree

‎NEXT_CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010

1111
### Bug Fixes
1212

13+
- Recover from concurrent OAuth token cache writes by using the fresh token stored by another process.
14+
1315
### Documentation
1416

1517
### Internal Changes

‎credentials/u2m/persistent_auth.go‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,18 @@ const (
4141
// token is proactively refreshed. This prevents callers from receiving
4242
// near-expired tokens that may expire before the next request.
4343
tokenRefreshBuffer = 5 * time.Minute
44+
45+
// Cache update recovery checks immediately and then waits for 25, 50, 100,
46+
// and 200 milliseconds. This gives a concurrent cache writer time to finish
47+
// while bounding the delay for persistent storage failures to 375 milliseconds.
48+
cacheUpdateRecoveryAttempts = 5
49+
50+
cacheUpdateRecoveryInitialDelay = 25 * time.Millisecond
51+
cacheUpdateRecoveryDelayFactor = 2
52+
53+
// Concurrent refreshes can finish at slightly different times, so their
54+
// expiration times need not be identical.
55+
cacheUpdateRecoveryExpiryDelta = time.Minute
4456
)
4557

4658
var (
@@ -316,6 +328,45 @@ func needsRefresh(t *oauth2.Token) bool {
316328
return !t.Expiry.IsZero() && time.Until(t.Expiry) < tokenRefreshBuffer
317329
}
318330

331+
// isFreshReplacement reports whether cached changed from old, is valid, and
332+
// does not expire significantly sooner than candidate.
333+
func isFreshReplacement(old, candidate, cached *oauth2.Token) bool {
334+
if cached.AccessToken == old.AccessToken || !cached.Valid() {
335+
return false
336+
}
337+
if cached.Expiry.IsZero() {
338+
return true
339+
}
340+
if candidate.Expiry.IsZero() {
341+
return false
342+
}
343+
return !cached.Expiry.Before(candidate.Expiry.Add(-cacheUpdateRecoveryExpiryDelta))
344+
}
345+
346+
// recoverCacheUpdate checks whether a concurrent cache update completed.
347+
// Retrying reads instead of writes avoids recreating the write race.
348+
func (a *PersistentAuth) recoverCacheUpdate(old, candidate *oauth2.Token) *oauth2.Token {
349+
delay := cacheUpdateRecoveryInitialDelay
350+
for attempt := 0; attempt < cacheUpdateRecoveryAttempts; attempt++ {
351+
if attempt > 0 {
352+
timer := time.NewTimer(delay)
353+
select {
354+
case <-a.ctx.Done():
355+
timer.Stop()
356+
return nil
357+
case <-timer.C:
358+
}
359+
delay *= cacheUpdateRecoveryDelayFactor
360+
}
361+
362+
cached, err := a.cache.Lookup(a.oAuthArgument.GetCacheKey())
363+
if err == nil && isFreshReplacement(old, candidate, cached) {
364+
return cached
365+
}
366+
}
367+
return nil
368+
}
369+
319370
// refresh refreshes the token for the given OAuthArgument, storing the new
320371
// token in the cache.
321372
//
@@ -382,6 +433,9 @@ func (a *PersistentAuth) refresh(oldToken *oauth2.Token) (*oauth2.Token, error)
382433
}
383434
err = a.cache.Store(a.oAuthArgument.GetCacheKey(), t)
384435
if err != nil {
436+
if cached := a.recoverCacheUpdate(oldToken, t); cached != nil {
437+
return cached, nil
438+
}
385439
return nil, fmt.Errorf("cache update: %w", err)
386440
}
387441
return t, nil

‎credentials/u2m/persistent_auth_test.go‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -618,6 +618,146 @@ func TestForceRefreshToken_RefreshesValidToken(t *testing.T) {
618618
}
619619
}
620620

621+
func TestForceRefreshToken_RecoversConcurrentCacheUpdate(t *testing.T) {
622+
now := time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC)
623+
old := &oauth2.Token{
624+
AccessToken: "old-access",
625+
RefreshToken: "old-refresh",
626+
Expiry: now.Add(time.Hour),
627+
}
628+
winner := &oauth2.Token{
629+
AccessToken: "winner-access",
630+
RefreshToken: "winner-refresh",
631+
Expiry: now.Add(time.Hour - 30*time.Second),
632+
}
633+
lookupCalls := 0
634+
c := &tokenCacheMock{
635+
lookup: func(key string) (*oauth2.Token, error) {
636+
lookupCalls++
637+
if lookupCalls <= 2 {
638+
return old, nil
639+
}
640+
return winner, nil
641+
},
642+
store: func(key string, tok *oauth2.Token) error {
643+
if tok.AccessToken != "candidate-access" {
644+
t.Fatalf("store(): want candidate access token, got %q", tok.AccessToken)
645+
}
646+
return errors.New("concurrent cache update")
647+
},
648+
}
649+
arg, err := NewBasicAccountOAuthArgument("https://accounts.cloud.databricks.com", "xyz")
650+
if err != nil {
651+
t.Fatalf("NewBasicAccountOAuthArgument(): %v", err)
652+
}
653+
p, err := NewPersistentAuth(
654+
context.Background(),
655+
WithTokenCache(c),
656+
WithHttpClient(&http.Client{
657+
Transport: fixtures.SliceTransport{
658+
{
659+
Method: "POST",
660+
Resource: "/oidc/accounts/xyz/v1/token",
661+
Response: `access_token=candidate-access&refresh_token=candidate-refresh&expires_in=3600`,
662+
ResponseHeaders: map[string][]string{
663+
"Content-Type": {"application/x-www-form-urlencoded"},
664+
},
665+
},
666+
},
667+
}),
668+
WithOAuthEndpointSupplier(MockOAuthEndpointSupplier{}),
669+
WithOAuthArgument(arg),
670+
)
671+
if err != nil {
672+
t.Fatalf("NewPersistentAuth(): %v", err)
673+
}
674+
defer p.Close()
675+
676+
tok, err := p.ForceRefreshToken()
677+
if err != nil {
678+
t.Fatalf("ForceRefreshToken(): want no error, got %v", err)
679+
}
680+
if tok.AccessToken != winner.AccessToken {
681+
t.Errorf("ForceRefreshToken(): want winner access token %q, got %q", winner.AccessToken, tok.AccessToken)
682+
}
683+
if tok.RefreshToken != "" {
684+
t.Errorf("ForceRefreshToken(): want refresh token redacted, got %q", tok.RefreshToken)
685+
}
686+
if lookupCalls != 3 {
687+
t.Errorf("Lookup(): want 3 calls, got %d", lookupCalls)
688+
}
689+
}
690+
691+
func TestIsFreshReplacement(t *testing.T) {
692+
now := time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC)
693+
old := &oauth2.Token{AccessToken: "old", Expiry: now.Add(time.Hour)}
694+
candidate := &oauth2.Token{AccessToken: "candidate", Expiry: now.Add(time.Hour)}
695+
696+
tests := []struct {
697+
name string
698+
candidate *oauth2.Token
699+
cached *oauth2.Token
700+
want bool
701+
}{
702+
{
703+
name: "same token",
704+
candidate: candidate,
705+
cached: &oauth2.Token{AccessToken: "old", Expiry: now.Add(time.Hour)},
706+
want: false,
707+
},
708+
{
709+
name: "expired replacement",
710+
candidate: candidate,
711+
cached: &oauth2.Token{AccessToken: "winner", Expiry: time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC)},
712+
want: false,
713+
},
714+
{
715+
name: "replacement expires too soon",
716+
candidate: candidate,
717+
cached: &oauth2.Token{AccessToken: "winner", Expiry: candidate.Expiry.Add(-time.Minute - time.Second)},
718+
want: false,
719+
},
720+
{
721+
name: "replacement expiry within tolerance",
722+
candidate: candidate,
723+
cached: &oauth2.Token{AccessToken: "winner", Expiry: candidate.Expiry.Add(-time.Minute)},
724+
want: true,
725+
},
726+
{
727+
name: "replacement expires later",
728+
candidate: candidate,
729+
cached: &oauth2.Token{AccessToken: "winner", Expiry: candidate.Expiry.Add(time.Minute)},
730+
want: true,
731+
},
732+
{
733+
name: "expiring candidate and non-expiring replacement",
734+
candidate: candidate,
735+
cached: &oauth2.Token{AccessToken: "winner"},
736+
want: true,
737+
},
738+
{
739+
name: "non-expiring candidate and replacement inside refresh buffer",
740+
candidate: &oauth2.Token{AccessToken: "candidate"},
741+
cached: &oauth2.Token{AccessToken: "winner", Expiry: now.Add(time.Minute)},
742+
want: false,
743+
},
744+
{
745+
name: "non-expiring candidate and replacement",
746+
candidate: &oauth2.Token{AccessToken: "candidate"},
747+
cached: &oauth2.Token{AccessToken: "winner"},
748+
want: true,
749+
},
750+
}
751+
752+
for _, tt := range tests {
753+
t.Run(tt.name, func(t *testing.T) {
754+
if got := isFreshReplacement(old, tt.candidate, tt.cached); got != tt.want {
755+
t.Errorf("isFreshReplacement(): want %t, got %t", tt.want, got)
756+
}
757+
})
758+
}
759+
}
760+
621761
func TestForceRefreshToken_WithInMemoryCachePreservesCachedRefreshToken(t *testing.T) {
622762
tokenCache := cache.NewInMemoryTokenCache()
623763
arg, err := NewBasicAccountOAuthArgument("https://accounts.cloud.databricks.com", "xyz")

0 commit comments

Comments
 (0)