Skip to content

Commit 5b6bb47

Browse files
committed
fix(kubeovn-webhook): retry certificate reload after a transient failure
A previous revision recorded the certificate file mtime even when the reload failed, to avoid re-reading a persistently bad file on every handshake. That conflated two failure modes: a transient, content- independent error (fd exhaustion, a torn read) on a VALID renewed file advanced the recorded mtime, so the staleness check never fired again and the renewed certificate was never loaded. The cached certificate then expired within renewBefore, and failurePolicy: Fail blocked all tenant pod creation - exactly the outage this reloader prevents. Advance the recorded mtime only on a successful load, and bound retries of a failed load with a wall-clock interval instead. A transient failure is now retried until it succeeds, while a persistently unreadable file is still retried only once per interval, not on every handshake. Keep a single reload in flight to avoid a thundering herd, and rate-limit the failure log. Add a regression test that fails when the mtime is advanced on failure. Detect change with mtime inequality, so a backward clock step is also picked up. Signed-off-by: IvanHunters <[email protected]>
1 parent a537bb6 commit 5b6bb47

2 files changed

Lines changed: 178 additions & 63 deletions

File tree

‎packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go‎

Lines changed: 86 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -24,66 +24,117 @@ type certReloader struct {
2424
certFile string
2525
keyFile string
2626

27+
// retryInterval bounds how often a failed reload (or a repeated stat failure) is
28+
// retried, so a persistently broken file does not read/parse/log on every handshake.
29+
retryInterval time.Duration
30+
2731
mu sync.RWMutex
2832
cert *tls.Certificate
2933
loadedModTime time.Time
34+
reloading bool
35+
nextRetry time.Time
36+
nextLog time.Time
3037
}
3138

39+
const defaultReloadRetryInterval = 30 * time.Second
40+
3241
// newCertReloader loads the initial key pair and returns a reloader for it.
3342
func newCertReloader(certFile, keyFile string) (*certReloader, error) {
34-
cr := &certReloader{certFile: certFile, keyFile: keyFile}
35-
if err := cr.reload(cr.certModTime()); err != nil {
43+
cr := &certReloader{
44+
certFile: certFile,
45+
keyFile: keyFile,
46+
retryInterval: defaultReloadRetryInterval,
47+
}
48+
modTime, err := fileModTime(certFile)
49+
if err != nil {
50+
return nil, err
51+
}
52+
if err := cr.load(modTime); err != nil {
3653
return nil, err
3754
}
3855
return cr, nil
3956
}
4057

41-
// certModTime returns the modification time of the certificate file, or the zero
42-
// time if it cannot be stat'd (logged so a broken mount is not silently invisible).
43-
func (cr *certReloader) certModTime() time.Time {
44-
fi, err := os.Stat(cr.certFile)
58+
// fileModTime returns the modification time of the given file.
59+
func fileModTime(name string) (time.Time, error) {
60+
fi, err := os.Stat(name)
4561
if err != nil {
46-
log.Printf("certificate stat failed, keeping previous certificate: %v", err)
47-
return time.Time{}
62+
return time.Time{}, err
4863
}
49-
return fi.ModTime()
64+
return fi.ModTime(), nil
5065
}
5166

52-
// reload reads the key pair from disk and atomically swaps the cached certificate.
53-
//
54-
// modTime is the certificate file's modification time observed BEFORE the read, so a
55-
// Secret swap racing the read leaves modTime older than the file on disk and is caught
56-
// on the next handshake instead of being cached under a newer mtime and missed. The
57-
// attempt is recorded regardless of outcome, so a persistently unreadable file is
58-
// retried only once its mtime advances, not on every handshake.
59-
func (cr *certReloader) reload(modTime time.Time) error {
67+
// load reads the key pair and, on success, caches it tagged with modTime. On failure
68+
// the cached certificate and its recorded mtime are left unchanged, so a transient
69+
// failure on a valid file is retried rather than the renewal being abandoned.
70+
func (cr *certReloader) load(modTime time.Time) error {
6071
cert, err := tls.LoadX509KeyPair(cr.certFile, cr.keyFile)
61-
72+
if err != nil {
73+
return err
74+
}
6275
cr.mu.Lock()
76+
cr.cert = &cert
6377
cr.loadedModTime = modTime
64-
if err == nil {
65-
cr.cert = &cert
66-
}
6778
cr.mu.Unlock()
68-
return err
79+
return nil
6980
}
7081

71-
// GetCertificate is a tls.Config.GetCertificate callback. It reloads the key pair when
72-
// the certificate file's modification time changes and keeps serving the last good
73-
// certificate if a reload fails (for example a torn read while the mounted Secret is
74-
// being updated).
75-
func (cr *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
76-
modTime := cr.certModTime()
82+
// refresh reloads the key pair when the certificate file's modification time changes.
83+
//
84+
// A failed reload keeps the last good certificate and does NOT advance the recorded
85+
// mtime, so a transient error (fd exhaustion, a torn read) is retried, at most once per
86+
// retryInterval, until it succeeds: a transient failure must never permanently pin an
87+
// expiring certificate. A single reload is kept in flight so a burst of handshakes does
88+
// not trigger a thundering herd of reads.
89+
func (cr *certReloader) refresh() {
90+
modTime, err := fileModTime(cr.certFile)
91+
if err != nil {
92+
cr.logRateLimited("certificate stat failed, keeping previous certificate: %v", err)
93+
return
94+
}
7795

78-
cr.mu.RLock()
79-
stale := !modTime.IsZero() && !modTime.Equal(cr.loadedModTime)
80-
cr.mu.RUnlock()
96+
cr.mu.Lock()
97+
if modTime.Equal(cr.loadedModTime) || cr.reloading || time.Now().Before(cr.nextRetry) {
98+
cr.mu.Unlock()
99+
return
100+
}
101+
cr.reloading = true
102+
cr.mu.Unlock()
103+
104+
loadErr := cr.load(modTime)
105+
106+
cr.mu.Lock()
107+
cr.reloading = false
108+
if loadErr == nil {
109+
cr.nextRetry = time.Time{}
110+
} else {
111+
cr.nextRetry = time.Now().Add(cr.retryInterval)
112+
}
113+
cr.mu.Unlock()
81114

82-
if stale {
83-
if err := cr.reload(modTime); err != nil {
84-
log.Printf("certificate reload failed, keeping previous certificate: %v", err)
85-
}
115+
if loadErr != nil {
116+
cr.logRateLimited("certificate reload failed, keeping previous certificate: %v", loadErr)
86117
}
118+
}
119+
120+
// logRateLimited emits a log line at most once per retryInterval, so a persistently
121+
// broken mount does not spam the log on every TLS handshake.
122+
func (cr *certReloader) logRateLimited(format string, args ...any) {
123+
cr.mu.Lock()
124+
if time.Now().Before(cr.nextLog) {
125+
cr.mu.Unlock()
126+
return
127+
}
128+
cr.nextLog = time.Now().Add(cr.retryInterval)
129+
cr.mu.Unlock()
130+
log.Printf(format, args...)
131+
}
132+
133+
// GetCertificate is a tls.Config.GetCertificate callback. It refreshes the key pair
134+
// when the certificate file changes and keeps serving the last good certificate if a
135+
// reload fails.
136+
func (cr *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
137+
cr.refresh()
87138

88139
cr.mu.RLock()
89140
defer cr.mu.RUnlock()

‎packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go‎

Lines changed: 92 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,8 @@ func genSelfSigned(t *testing.T, serial int64) (certPEM, keyPEM []byte) {
5050
return certPEM, keyPEM
5151
}
5252

53-
// writeKeyPair writes the cert/key files and bumps their mtime strictly forward,
54-
// mirroring how cert-manager replaces the mounted Secret on renewal.
53+
// writeKeyPair writes the cert/key files and sets their mtime, mirroring how
54+
// cert-manager replaces the mounted Secret on renewal.
5555
func writeKeyPair(t *testing.T, certFile, keyFile string, certPEM, keyPEM []byte, mtime time.Time) {
5656
t.Helper()
5757
if err := os.WriteFile(certFile, certPEM, 0o600); err != nil {
@@ -60,12 +60,36 @@ func writeKeyPair(t *testing.T, certFile, keyFile string, certPEM, keyPEM []byte
6060
if err := os.WriteFile(keyFile, keyPEM, 0o600); err != nil {
6161
t.Fatalf("write key: %v", err)
6262
}
63-
if err := os.Chtimes(certFile, mtime, mtime); err != nil {
64-
t.Fatalf("chtimes cert: %v", err)
63+
setModTime(t, certFile, mtime)
64+
setModTime(t, keyFile, mtime)
65+
}
66+
67+
func setModTime(t *testing.T, name string, mtime time.Time) {
68+
t.Helper()
69+
if err := os.Chtimes(name, mtime, mtime); err != nil {
70+
t.Fatalf("chtimes %s: %v", name, err)
6571
}
66-
if err := os.Chtimes(keyFile, mtime, mtime); err != nil {
67-
t.Fatalf("chtimes key: %v", err)
72+
}
73+
74+
// certSerial returns the serial number of a *tls.Certificate leaf without mutating the
75+
// shared value returned by GetCertificate (it parses into a local when Leaf is unset).
76+
func certSerial(t *testing.T, c *tls.Certificate) int64 {
77+
t.Helper()
78+
if c == nil {
79+
t.Fatal("nil certificate")
80+
}
81+
leaf := c.Leaf
82+
if leaf == nil {
83+
if len(c.Certificate) == 0 {
84+
t.Fatal("certificate has no DER data")
85+
}
86+
parsed, err := x509.ParseCertificate(c.Certificate[0])
87+
if err != nil {
88+
t.Fatalf("parse served cert: %v", err)
89+
}
90+
leaf = parsed
6891
}
92+
return leaf.SerialNumber.Int64()
6993
}
7094

7195
// servedSerial completes a TLS handshake against addr and returns the serial number
@@ -84,10 +108,9 @@ func servedSerial(t *testing.T, addr string) int64 {
84108
return certs[0].SerialNumber.Int64()
85109
}
86110

87-
// TestReloadingTLSConfigServesRenewedCertificate is the regression test for the
88-
// cert-manager renewal not being picked up: it fails against the old code that
89-
// loads the key pair once into tls.Config.Certificates, and passes once the
90-
// certificate is served via a reloading GetCertificate callback.
111+
// TestReloadingTLSConfigServesRenewedCertificate is the core regression test: it fails
112+
// against the old code that loads the key pair once into tls.Config.Certificates, and
113+
// passes once the certificate is served via a reloading GetCertificate callback.
91114
func TestReloadingTLSConfigServesRenewedCertificate(t *testing.T) {
92115
dir := t.TempDir()
93116
certFile := filepath.Join(dir, "tls.crt")
@@ -136,9 +159,9 @@ func TestReloadingTLSConfigServesRenewedCertificate(t *testing.T) {
136159
}
137160
}
138161

139-
// TestCertReloaderKeepsLastGoodCertificate verifies that a failed reload (e.g. a
140-
// torn read while the volume is updated) does not take the webhook down: it keeps
141-
// serving the previously loaded certificate instead of erroring the handshake.
162+
// TestCertReloaderKeepsLastGoodCertificate verifies that a failed reload (e.g. a torn
163+
// read while the volume is updated) does not take the webhook down: it keeps serving
164+
// the previously loaded certificate instead of erroring the handshake.
142165
func TestCertReloaderKeepsLastGoodCertificate(t *testing.T) {
143166
dir := t.TempDir()
144167
certFile := filepath.Join(dir, "tls.crt")
@@ -156,26 +179,67 @@ func TestCertReloaderKeepsLastGoodCertificate(t *testing.T) {
156179
if err := os.WriteFile(certFile, []byte("not a certificate"), 0o600); err != nil {
157180
t.Fatalf("corrupt cert: %v", err)
158181
}
159-
if err := os.Chtimes(certFile, time.Now().Add(2*time.Second), time.Now().Add(2*time.Second)); err != nil {
160-
t.Fatalf("chtimes: %v", err)
161-
}
182+
setModTime(t, certFile, time.Now().Add(2*time.Second))
162183

163184
got, err := cr.GetCertificate(&tls.ClientHelloInfo{})
164185
if err != nil {
165186
t.Fatalf("GetCertificate returned error instead of serving last good cert: %v", err)
166187
}
167-
if got == nil || got.Leaf == nil {
168-
// Leaf may be nil depending on the Go version; fall back to parsing.
169-
if got == nil || len(got.Certificate) == 0 {
170-
t.Fatalf("GetCertificate returned no certificate")
171-
}
172-
leaf, perr := x509.ParseCertificate(got.Certificate[0])
173-
if perr != nil {
174-
t.Fatalf("parse served cert: %v", perr)
175-
}
176-
got.Leaf = leaf
188+
if certSerial(t, got) != 1 {
189+
t.Fatalf("expected to keep serving serial 1 after failed reload, got %d", certSerial(t, got))
190+
}
191+
}
192+
193+
// TestCertReloaderRetriesAfterTransientFailure is the regression test for the reloader
194+
// abandoning a renewal on a transient error. A failed reload must NOT advance the
195+
// recorded mtime, so once the (unchanged-mtime) file becomes readable the renewed
196+
// certificate is still picked up. Against the code that advanced the mtime on failure,
197+
// the renewal is permanently missed and this test fails.
198+
func TestCertReloaderRetriesAfterTransientFailure(t *testing.T) {
199+
dir := t.TempDir()
200+
certFile := filepath.Join(dir, "tls.crt")
201+
keyFile := filepath.Join(dir, "tls.key")
202+
203+
certA, keyA := genSelfSigned(t, 1)
204+
writeKeyPair(t, certFile, keyFile, certA, keyA, time.Now().Add(-2*time.Second))
205+
206+
cr, err := newCertReloader(certFile, keyFile)
207+
if err != nil {
208+
t.Fatalf("newCertReloader: %v", err)
209+
}
210+
cr.retryInterval = 0 // retry immediately, no backoff wait in the test
211+
212+
// Renewal advances the mtime, but the first load fails (simulating a transient,
213+
// content-independent error) by writing an unreadable cert at the new mtime.
214+
renewMod := time.Now().Add(2 * time.Second)
215+
if err := os.WriteFile(certFile, []byte("transiently unreadable"), 0o600); err != nil {
216+
t.Fatalf("write bad cert: %v", err)
217+
}
218+
setModTime(t, certFile, renewMod)
219+
220+
if got, gerr := cr.GetCertificate(&tls.ClientHelloInfo{}); gerr != nil {
221+
t.Fatalf("GetCertificate errored during transient failure: %v", gerr)
222+
} else if certSerial(t, got) != 1 {
223+
t.Fatalf("during transient failure: expected to keep serial 1, got %d", certSerial(t, got))
224+
}
225+
226+
// The renewed, valid certificate is now readable at the SAME mtime as the failed
227+
// attempt. The reloader must still pick it up on retry.
228+
certB, keyB := genSelfSigned(t, 2)
229+
if err := os.WriteFile(certFile, certB, 0o600); err != nil {
230+
t.Fatalf("write renewed cert: %v", err)
231+
}
232+
if err := os.WriteFile(keyFile, keyB, 0o600); err != nil {
233+
t.Fatalf("write renewed key: %v", err)
234+
}
235+
setModTime(t, certFile, renewMod)
236+
setModTime(t, keyFile, renewMod)
237+
238+
got, gerr := cr.GetCertificate(&tls.ClientHelloInfo{})
239+
if gerr != nil {
240+
t.Fatalf("GetCertificate errored after retry: %v", gerr)
177241
}
178-
if got.Leaf.SerialNumber.Int64() != 1 {
179-
t.Fatalf("expected to keep serving serial 1 after failed reload, got %d", got.Leaf.SerialNumber.Int64())
242+
if certSerial(t, got) != 2 {
243+
t.Fatalf("after transient failure: expected renewed serial 2 on retry, got %d (mtime advanced on failure, renewal permanently missed)", certSerial(t, got))
180244
}
181245
}

0 commit comments

Comments
 (0)