fix(auth): retry stale scoped token once after 403

clawsweeper[bot] · clawsweeper[bot] · commit fb5d8f56058f · 2026-06-30T05:43:29.000Z
diff --git a/internal/googleapi/client.go b/internal/googleapi/client.go
@@ -148,10 +148,18 @@ func authenticatedTransportWithStoredScopeCheck(
 		}
 	}
 
-	return readOnlyTransportFromContext(ctx, NewRetryTransport(&oauth2.Transport{
+	retryTransport := NewRetryTransport(&oauth2.Transport{
 		Source: ts,
 		Base:   newBaseTransport(),
-	})), nil
+	})
+
+	if refresher, ok := ts.(interface {
+		ForceRefresh(context.Context) error
+	}); ok {
+		retryTransport.RefreshAuth = refresher.ForceRefresh
+	}
+
+	return readOnlyTransportFromContext(ctx, retryTransport), nil
 }
 
 func optionsForAccountScopes(ctx context.Context, serviceLabel string, email string, scopes []string) ([]option.ClientOption, error) {
diff --git a/internal/googleapi/client_auth.go b/internal/googleapi/client_auth.go
@@ -35,10 +35,78 @@ type persistingTokenSource struct {
 	tok secrets.Token
 }
 
+type forceRefreshTokenSource interface {
+	ForceRefresh(context.Context) (*oauth2.Token, error)
+}
+
+var (
+	errBaseTokenSourceCannotForceRefresh = errors.New("base token source cannot force refresh")
+	errBaseTokenSourceReturnedNilToken   = errors.New("base token source returned nil token")
+)
+
+type resettableOAuthTokenSource struct {
+	mu           sync.Mutex
+	source       oauth2.TokenSource
+	newSource    func(*oauth2.Token) oauth2.TokenSource
+	refreshToken string
+}
+
 type tokenAliasDeleter interface {
 	DeleteTokenAlias(client string, email string) error
 }
 
+func newResettableOAuthTokenSource(newSource func(*oauth2.Token) oauth2.TokenSource, initial *oauth2.Token) *resettableOAuthTokenSource {
+	return &resettableOAuthTokenSource{
+		source:       newSource(initial),
+		newSource:    newSource,
+		refreshToken: strings.TrimSpace(initial.RefreshToken),
+	}
+}
+
+func (r *resettableOAuthTokenSource) Token() (*oauth2.Token, error) {
+	r.mu.Lock()
+	source := r.source
+	r.mu.Unlock()
+
+	t, err := source.Token()
+	if err != nil {
+		return nil, fmt.Errorf("resettable oauth token source: %w", err)
+	}
+
+	r.rememberRefreshToken(t)
+
+	return t, nil
+}
+
+func (r *resettableOAuthTokenSource) ForceRefresh(context.Context) (*oauth2.Token, error) {
+	r.mu.Lock()
+	refreshToken := r.refreshToken
+	r.source = r.newSource(&oauth2.Token{RefreshToken: refreshToken})
+	source := r.source
+	r.mu.Unlock()
+
+	t, err := source.Token()
+	if err != nil {
+		return nil, fmt.Errorf("resettable oauth token source refresh: %w", err)
+	}
+
+	r.rememberRefreshToken(t)
+
+	return t, nil
+}
+
+func (r *resettableOAuthTokenSource) rememberRefreshToken(t *oauth2.Token) {
+	if t == nil {
+		return
+	}
+
+	if refreshToken := strings.TrimSpace(t.RefreshToken); refreshToken != "" {
+		r.mu.Lock()
+		r.refreshToken = refreshToken
+		r.mu.Unlock()
+	}
+}
+
 func newPersistingTokenSource(base oauth2.TokenSource, store secrets.Store, client string, email string, tok secrets.Token, serviceLabel string, updateEmailReferences googleauth.EmailReferenceUpdater) oauth2.TokenSource {
 	return &persistingTokenSource{
 		base:                  base,
@@ -57,6 +125,30 @@ func (p *persistingTokenSource) Token() (*oauth2.Token, error) {
 		return nil, fmt.Errorf("base token source: %w", err)
 	}
 
+	return p.persistToken(t)
+}
+
+func (p *persistingTokenSource) ForceRefresh(ctx context.Context) error {
+	refresher, ok := p.base.(forceRefreshTokenSource)
+	if !ok {
+		return errBaseTokenSourceCannotForceRefresh
+	}
+
+	t, err := refresher.ForceRefresh(ctx)
+	if err != nil {
+		return fmt.Errorf("force token refresh: %w", err)
+	}
+
+	_, err = p.persistToken(t)
+
+	return err
+}
+
+func (p *persistingTokenSource) persistToken(t *oauth2.Token) (*oauth2.Token, error) {
+	if t == nil {
+		return nil, errBaseTokenSourceReturnedNilToken
+	}
+
 	refreshToken := strings.TrimSpace(t.RefreshToken)
 
 	p.mu.Lock()
@@ -126,26 +218,26 @@ func (p *persistingTokenSource) Token() (*oauth2.Token, error) {
 	}
 
 	if err := p.store.SetToken(p.client, persistEmail, updated); err != nil {
-		slog.Warn("persist refreshed token metadata failed", "email", persistEmail, "client", p.client, "err", err)
+		slog.Warn("persist refreshed token metadata failed", "email", persistEmail, "client", p.client, "err", err) //nolint:gosec // logged values are token metadata identifiers for auth diagnostics
 		return t, nil
 	}
 
 	if !strings.EqualFold(p.email, persistEmail) {
 		if err := googleauth.MigrateStoredEmailReferences(p.store, p.updateEmailReferences, p.client, p.email, persistEmail); err != nil {
-			slog.Warn("migrate renamed token email references failed", "old_email", p.email, "new_email", persistEmail, "client", p.client, "err", err)
+			slog.Warn("migrate renamed token email references failed", "old_email", p.email, "new_email", persistEmail, "client", p.client, "err", err) //nolint:gosec // logged values are token metadata identifiers for auth diagnostics
 		}
 
 		aliasDeleter, ok := p.store.(tokenAliasDeleter)
 		if !ok {
-			slog.Debug("token store cannot delete renamed email alias", "old_email", p.email, "new_email", persistEmail, "client", p.client)
+			slog.Debug("token store cannot delete renamed email alias", "old_email", p.email, "new_email", persistEmail, "client", p.client) //nolint:gosec // logged values are token metadata identifiers for auth diagnostics
 		} else if err := aliasDeleter.DeleteTokenAlias(p.client, p.email); err != nil {
-			slog.Warn("delete renamed token alias failed", "old_email", p.email, "new_email", persistEmail, "client", p.client, "err", err)
+			slog.Warn("delete renamed token alias failed", "old_email", p.email, "new_email", persistEmail, "client", p.client, "err", err) //nolint:gosec // logged values are token metadata identifiers for auth diagnostics
 		}
 	}
 
 	p.tok = updated
 	p.email = persistEmail
-	slog.Debug("persisted refreshed token metadata", "email", persistEmail, "client", p.client)
+	slog.Debug("persisted refreshed token metadata", "email", persistEmail, "client", p.client) //nolint:gosec // logged values are token metadata identifiers for auth diagnostics
 
 	return t, nil
 }
@@ -276,7 +368,9 @@ func tokenSourceForAccountScopesWithStoredScopeCheck(
 	// Ensure refresh-token exchanges don't hang forever.
 	ctx = context.WithValue(ctx, oauth2.HTTPClient, &http.Client{Timeout: tokenExchangeTimeout})
 
-	baseSource := cfg.TokenSource(ctx, &oauth2.Token{
+	baseSource := newResettableOAuthTokenSource(func(t *oauth2.Token) oauth2.TokenSource {
+		return cfg.TokenSource(ctx, t)
+	}, &oauth2.Token{
 		RefreshToken: tok.RefreshToken,
 		AccessToken:  strings.TrimSpace(tok.AccessToken),
 		Expiry:       tok.AccessTokenExpiresAt,
diff --git a/internal/googleapi/transport.go b/internal/googleapi/transport.go
@@ -11,10 +11,14 @@ import (
 	"math/big"
 	"net/http"
 	"strconv"
+	"strings"
 	"time"
 )
 
-const maxBufferedReplayBodyBytes = int64(16 << 20)
+const (
+	maxBufferedReplayBodyBytes    = int64(16 << 20)
+	maxAuthRetryResponseBodyBytes = int64(1 << 20)
+)
 
 var errRequestBodyTooLarge = errors.New("request body too large to buffer for retry")
 
@@ -26,6 +30,7 @@ type RetryTransport struct {
 	MaxRetries5xx  int
 	BaseDelay      time.Duration
 	CircuitBreaker *CircuitBreaker
+	RefreshAuth    func(context.Context) error
 }
 
 // NewRetryTransport creates a RetryTransport with sensible defaults.
@@ -57,6 +62,7 @@ func (t *RetryTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 	var resp *http.Response
 	retries429 := 0
 	retries5xx := 0
+	retriedAuth := false
 
 	for {
 		// Reset body for retry
@@ -134,6 +140,28 @@ func (t *RetryTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 			continue
 		}
 
+		if resp.StatusCode == http.StatusForbidden && t.RefreshAuth != nil && !retriedAuth && replayable {
+			insufficientScopes, detectErr := responseIndicatesInsufficientScopes(resp)
+			if detectErr != nil {
+				slog.Debug("could not inspect auth failure response for retry", "err", detectErr)
+				return resp, nil
+			}
+
+			if insufficientScopes {
+				slog.Debug("insufficient scopes response, refreshing auth token and retrying")
+
+				drainAndClose(resp.Body)
+
+				if err := t.RefreshAuth(req.Context()); err != nil {
+					return nil, fmt.Errorf("refresh auth after insufficient scopes response: %w", err)
+				}
+
+				retriedAuth = true
+
+				continue
+			}
+		}
+
 		// Other errors (4xx except 429): don't retry
 		return resp, nil
 	}
@@ -249,3 +277,32 @@ func drainAndClose(body io.ReadCloser) {
 	_, _ = io.Copy(io.Discard, io.LimitReader(body, 1<<20))
 	_ = body.Close()
 }
+
+func responseIndicatesInsufficientScopes(resp *http.Response) (bool, error) {
+	if resp == nil || resp.Body == nil {
+		return false, nil
+	}
+
+	if resp.ContentLength > maxAuthRetryResponseBodyBytes {
+		return false, nil
+	}
+
+	bodyBytes, err := io.ReadAll(io.LimitReader(resp.Body, maxAuthRetryResponseBodyBytes+1))
+	_ = resp.Body.Close()
+	resp.Body = io.NopCloser(bytes.NewReader(bodyBytes))
+
+	if err != nil {
+		return false, fmt.Errorf("read auth failure response: %w", err)
+	}
+
+	if int64(len(bodyBytes)) > maxAuthRetryResponseBodyBytes {
+		return false, nil
+	}
+
+	text := strings.ToLower(string(bodyBytes))
+	normalized := strings.NewReplacer("_", " ", "-", " ").Replace(text)
+
+	return strings.Contains(normalized, "insufficient authentication scopes") ||
+		strings.Contains(normalized, "insufficient scopes") ||
+		strings.Contains(normalized, "access token scope insufficient"), nil
+}
diff --git a/internal/googleapi/transport_more_test.go b/internal/googleapi/transport_more_test.go

Original file line number	Diff line number	Diff line change
`@@ -148,10 +148,18 @@ func authenticatedTransportWithStoredScopeCheck(`
`148`	`148`	`}`
`149`	`149`	`}`
`150`	`150`
`151`		`- return readOnlyTransportFromContext(ctx, NewRetryTransport(&oauth2.Transport{`
	`151`	`+ retryTransport := NewRetryTransport(&oauth2.Transport{`
`152`	`152`	`Source: ts,`
`153`	`153`	`Base: newBaseTransport(),`
`154`		`- })), nil`
	`154`	`+ })`
	`155`	`+`
	`156`	`+ if refresher, ok := ts.(interface {`
	`157`	`+ ForceRefresh(context.Context) error`
	`158`	`+ }); ok {`
	`159`	`+ retryTransport.RefreshAuth = refresher.ForceRefresh`
	`160`	`+ }`
	`161`	`+`
	`162`	`+ return readOnlyTransportFromContext(ctx, retryTransport), nil`
`155`	`163`	`}`
`156`	`164`
`157`	`165`	`func optionsForAccountScopes(ctx context.Context, serviceLabel string, email string, scopes []string) ([]option.ClientOption, error) {`