Skip to content

Commit 3b31bbb

Browse files
mnenciaarmru
andauthored
fix(backup): retry volume snapshot backups on transient instance-manager connection errors (#11069)
A transport-level failure to reach the instance manager, for example a dial timeout caused by a brief pod-network disruption, was treated as a terminal error and failed a volume snapshot backup even when its snapshots were already provisioned. `isNetworkErrorRetryable`, which already requeues operator to instance-manager communication errors, did not recognize a `net.Error`, so the failure escaped as fatal during the finalize step. Detect `net.Error` there so these failures requeue and the backup is retried instead of discarded. Closes #11068 Signed-off-by: Marco Nenciarini <[email protected]> Signed-off-by: Armando Ruocco <[email protected]> Co-authored-by: Armando Ruocco <[email protected]>
1 parent 19d9494 commit 3b31bbb

2 files changed

Lines changed: 38 additions & 2 deletions

File tree

‎pkg/reconciler/backup/volumesnapshot/errors.go‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ package volumesnapshot
2222
import (
2323
"context"
2424
"errors"
25+
"net"
2526
"regexp"
2627
"strconv"
2728
"strings"
@@ -36,7 +37,7 @@ var (
3637
httpStatusCodeRegex = regexp.MustCompile(`HTTPStatusCode:\s(\d{3})`)
3738
)
3839

39-
// isErrorRetryable detects is an error is retryable or not.
40+
// isNetworkErrorRetryable detects if an error is retryable or not.
4041
//
4142
// Important: this function is intended for detecting errors that
4243
// occur during communication between the operator and the Kubernetes
@@ -45,8 +46,24 @@ var (
4546
// It is not designed to check errors raised by the CSI driver and
4647
// exposed by the CSI snapshotter sidecar.
4748
func isNetworkErrorRetryable(err error) bool {
49+
// A transport-level failure to reach the instance manager surfaces as a
50+
// net.Error. The HTTP client wraps every such failure in a *net/url.Error,
51+
// which itself satisfies net.Error, so this matches all of them: dial
52+
// timeout, connection refused or reset, DNS failure, and TLS or certificate
53+
// errors. In each case the connection never produced an authenticated
54+
// response, so requeuing is safe. It matters most in the finalize step,
55+
// where the snapshots are already provisioned and a single failure would
56+
// otherwise discard an otherwise complete backup.
57+
//
58+
// This is intentionally the opposite of the in-request retry in
59+
// remote.getReplicaStatusFromPodViaHTTP, which treats a net.Error timeout as
60+
// non-retryable: that path retries within a single reconcile, whereas here we
61+
// requeue the whole reconcile.
62+
var netErr net.Error
63+
4864
return apierrs.IsServerTimeout(err) || apierrs.IsConflict(err) || apierrs.IsInternalError(err) ||
49-
errors.Is(err, context.DeadlineExceeded) || remote.IsTransientAuthError(err)
65+
errors.Is(err, context.DeadlineExceeded) || remote.IsTransientAuthError(err) ||
66+
errors.As(err, &netErr)
5067
}
5168

5269
// isCSIErrorMessageRetriable detects if a certain error message

‎pkg/reconciler/backup/volumesnapshot/errors_test.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"context"
2424
"errors"
2525
"fmt"
26+
"net"
2627
"net/http"
2728

2829
apierrs "k8s.io/apimachinery/pkg/api/errors"
@@ -104,6 +105,24 @@ var _ = Describe("isNetworkErrorRetryable", func() {
104105
Expect(isNetworkErrorRetryable(err)).To(BeTrue())
105106
})
106107

108+
It("retries a transport-level dial timeout to the instance manager (production path)", func() {
109+
// Mirrors the error chain produced when the finalize status read cannot reach
110+
// the instance manager: a *net.OpError wrapped by the HTTP client and the
111+
// reconciler.
112+
err := fmt.Errorf("while getting status while finalizing: %w",
113+
fmt.Errorf("while executing http request: %w",
114+
&net.OpError{Op: "dial", Net: "tcp", Err: errors.New("i/o timeout")}))
115+
Expect(isNetworkErrorRetryable(err)).To(BeTrue())
116+
})
117+
118+
It("retries a transport-level connection error", func() {
119+
// The bare, unwrapped net.OpError, and a connection refused rather than a
120+
// timeout: errors.As must match the type directly, without relying on a
121+
// Timeout()/Temporary() check.
122+
err := &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")}
123+
Expect(isNetworkErrorRetryable(err)).To(BeTrue())
124+
})
125+
107126
It("does not retry a 401 (no client certificate, e.g. non-TLS status port)", func() {
108127
err := &remote.StatusError{StatusCode: http.StatusUnauthorized, Body: "client certificate required"}
109128
Expect(isNetworkErrorRetryable(err)).To(BeFalse())

0 commit comments

Comments
 (0)