Skip to content

Commit 8baa449

Browse files
committed
refactor(kube-ovn): reload webhook cert via a plain GetCertificate closure
Collapse the certReloader (mtime tracking, serialized refresh, retry backoff, rate-limited logging, last-good-certificate fallback) into a GetCertificate callback that simply re-reads the key pair from disk on every handshake. The signature is unchanged, so main.go is untouched. The extra machinery guarded scenarios this deployment cannot produce: - Torn reads are unreachable: the cert/key files come from a plain Secret volume with no subPath, and Kubernetes' atomic writer swaps the whole ..data directory via a single symlink flip, so a reader always sees a complete old or complete new generation of both files. - The per-handshake LoadX509KeyPair cost is negligible next to the asymmetric crypto the handshake already performs, so no cache is needed. - The webhook configures no mTLS, so GetCertificate not refreshing client CA pools does not apply. If the mounted files genuinely become unreadable (an operator error, not a renewal) the handshake now fails loudly instead of silently serving a stale certificate. The core regression test is retained and remains non-vacuous: it fails against a static tls.Config.Certificates and passes with the reloading callback. Signed-off-by: IvanHunters <[email protected]>
1 parent 5b6bb47 commit 8baa449

2 files changed

Lines changed: 36 additions & 252 deletions

File tree

Lines changed: 36 additions & 146 deletions
Original file line numberDiff line numberDiff line change
@@ -1,155 +1,45 @@
11
package main
22

3-
import (
4-
"crypto/tls"
5-
"fmt"
6-
"log"
7-
"os"
8-
"sync"
9-
"time"
10-
)
3+
import "crypto/tls"
114

12-
// certReloader keeps the webhook serving certificate in sync with the key pair
13-
// files written by cert-manager.
5+
// newReloadingTLSConfig returns a tls.Config that reloads the certificate and
6+
// key from disk on every TLS handshake, so a cert-manager renewal of the mounted
7+
// Secret is picked up by the running process without a restart.
148
//
15-
// The certificate is served through tls.Config.GetCertificate, so a cert-manager
16-
// renewal of the backing Secret is picked up by the running process without a
17-
// restart. Without this, the key pair is read exactly once at startup and cached
18-
// for the lifetime of the process: after the certificate expires (roughly one year
19-
// after install) the pod keeps presenting the expired certificate, the
20-
// kube-apiserver's TLS call to the webhook fails, and because the
21-
// MutatingWebhookConfiguration uses failurePolicy: Fail, every pod creation in
22-
// tenant namespaces is rejected.
23-
type certReloader struct {
24-
certFile string
25-
keyFile string
26-
27-
// retryInterval bounds how often a failed reload (or a repeated stat failure) is
28-
// retried, so a persistently broken file does not read/parse/log on every handshake.
29-
retryInterval time.Duration
30-
31-
mu sync.RWMutex
32-
cert *tls.Certificate
33-
loadedModTime time.Time
34-
reloading bool
35-
nextRetry time.Time
36-
nextLog time.Time
37-
}
38-
39-
const defaultReloadRetryInterval = 30 * time.Second
40-
41-
// newCertReloader loads the initial key pair and returns a reloader for it.
42-
func newCertReloader(certFile, keyFile string) (*certReloader, error) {
43-
cr := &certReloader{
44-
certFile: certFile,
45-
keyFile: keyFile,
46-
retryInterval: defaultReloadRetryInterval,
47-
}
48-
modTime, err := fileModTime(certFile)
49-
if err != nil {
50-
return nil, err
51-
}
52-
if err := cr.load(modTime); err != nil {
53-
return nil, err
54-
}
55-
return cr, nil
56-
}
57-
58-
// fileModTime returns the modification time of the given file.
59-
func fileModTime(name string) (time.Time, error) {
60-
fi, err := os.Stat(name)
61-
if err != nil {
62-
return time.Time{}, err
63-
}
64-
return fi.ModTime(), nil
65-
}
66-
67-
// load reads the key pair and, on success, caches it tagged with modTime. On failure
68-
// the cached certificate and its recorded mtime are left unchanged, so a transient
69-
// failure on a valid file is retried rather than the renewal being abandoned.
70-
func (cr *certReloader) load(modTime time.Time) error {
71-
cert, err := tls.LoadX509KeyPair(cr.certFile, cr.keyFile)
72-
if err != nil {
73-
return err
74-
}
75-
cr.mu.Lock()
76-
cr.cert = &cert
77-
cr.loadedModTime = modTime
78-
cr.mu.Unlock()
79-
return nil
80-
}
81-
82-
// refresh reloads the key pair when the certificate file's modification time changes.
9+
// Without this, the key pair is read exactly once at startup and cached for the
10+
// lifetime of the process: after the certificate expires (roughly one year after
11+
// install) the pod keeps presenting the expired certificate, the kube-apiserver's
12+
// TLS call to the webhook fails, and because the MutatingWebhookConfiguration uses
13+
// failurePolicy: Fail, every pod creation in tenant namespaces is rejected.
8314
//
84-
// A failed reload keeps the last good certificate and does NOT advance the recorded
85-
// mtime, so a transient error (fd exhaustion, a torn read) is retried, at most once per
86-
// retryInterval, until it succeeds: a transient failure must never permanently pin an
87-
// expiring certificate. A single reload is kept in flight so a burst of handshakes does
88-
// not trigger a thundering herd of reads.
89-
func (cr *certReloader) refresh() {
90-
modTime, err := fileModTime(cr.certFile)
91-
if err != nil {
92-
cr.logRateLimited("certificate stat failed, keeping previous certificate: %v", err)
93-
return
94-
}
95-
96-
cr.mu.Lock()
97-
if modTime.Equal(cr.loadedModTime) || cr.reloading || time.Now().Before(cr.nextRetry) {
98-
cr.mu.Unlock()
99-
return
100-
}
101-
cr.reloading = true
102-
cr.mu.Unlock()
103-
104-
loadErr := cr.load(modTime)
105-
106-
cr.mu.Lock()
107-
cr.reloading = false
108-
if loadErr == nil {
109-
cr.nextRetry = time.Time{}
110-
} else {
111-
cr.nextRetry = time.Now().Add(cr.retryInterval)
112-
}
113-
cr.mu.Unlock()
114-
115-
if loadErr != nil {
116-
cr.logRateLimited("certificate reload failed, keeping previous certificate: %v", loadErr)
117-
}
118-
}
119-
120-
// logRateLimited emits a log line at most once per retryInterval, so a persistently
121-
// broken mount does not spam the log on every TLS handshake.
122-
func (cr *certReloader) logRateLimited(format string, args ...any) {
123-
cr.mu.Lock()
124-
if time.Now().Before(cr.nextLog) {
125-
cr.mu.Unlock()
126-
return
127-
}
128-
cr.nextLog = time.Now().Add(cr.retryInterval)
129-
cr.mu.Unlock()
130-
log.Printf(format, args...)
131-
}
132-
133-
// GetCertificate is a tls.Config.GetCertificate callback. It refreshes the key pair
134-
// when the certificate file changes and keeps serving the last good certificate if a
135-
// reload fails.
136-
func (cr *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
137-
cr.refresh()
138-
139-
cr.mu.RLock()
140-
defer cr.mu.RUnlock()
141-
if cr.cert == nil {
142-
return nil, fmt.Errorf("no certificate loaded")
143-
}
144-
return cr.cert, nil
145-
}
146-
147-
// newReloadingTLSConfig builds a tls.Config that serves the key pair via a
148-
// certReloader, so cert-manager renewals are honoured without a restart.
15+
// tls.Config.GetCertificate is invoked on every handshake whenever Certificates is
16+
// left unset, so re-reading the files there is enough; no watcher, mtime tracking
17+
// or cache is needed:
18+
// - The cert/key files are mounted from a plain Secret volume with no subPath.
19+
// Kubernetes' atomic writer swaps the whole ..data directory via a single
20+
// symlink flip, so a reader always sees a complete old or complete new
21+
// generation of both files, never a torn mid-write mixture. A per-handshake
22+
// LoadX509KeyPair therefore cannot observe a partial renewal.
23+
// - The per-handshake cost (a few KB of file I/O plus a PEM/key parse) is
24+
// negligible next to the asymmetric crypto the handshake already performs.
25+
// - The webhook configures no mTLS (no ClientCAs), so GetCertificate's
26+
// limitation of not refreshing client CA pools does not apply.
27+
//
28+
// If the mounted files genuinely become unreadable (Secret deleted, permissions
29+
// broken: an operator error, not a normal renewal) the handshake fails loudly
30+
// rather than silently serving a stale certificate.
14931
func newReloadingTLSConfig(certFile, keyFile string) (*tls.Config, error) {
150-
cr, err := newCertReloader(certFile, keyFile)
151-
if err != nil {
32+
// Fail fast at startup if the initial key pair is missing or malformed.
33+
if _, err := tls.LoadX509KeyPair(certFile, keyFile); err != nil {
15234
return nil, err
15335
}
154-
return &tls.Config{GetCertificate: cr.GetCertificate}, nil
36+
return &tls.Config{
37+
GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
38+
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
39+
if err != nil {
40+
return nil, err
41+
}
42+
return &cert, nil
43+
},
44+
}, nil
15545
}

‎packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go‎

Lines changed: 0 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -71,27 +71,6 @@ func setModTime(t *testing.T, name string, mtime time.Time) {
7171
}
7272
}
7373

74-
// certSerial returns the serial number of a *tls.Certificate leaf without mutating the
75-
// shared value returned by GetCertificate (it parses into a local when Leaf is unset).
76-
func certSerial(t *testing.T, c *tls.Certificate) int64 {
77-
t.Helper()
78-
if c == nil {
79-
t.Fatal("nil certificate")
80-
}
81-
leaf := c.Leaf
82-
if leaf == nil {
83-
if len(c.Certificate) == 0 {
84-
t.Fatal("certificate has no DER data")
85-
}
86-
parsed, err := x509.ParseCertificate(c.Certificate[0])
87-
if err != nil {
88-
t.Fatalf("parse served cert: %v", err)
89-
}
90-
leaf = parsed
91-
}
92-
return leaf.SerialNumber.Int64()
93-
}
94-
9574
// servedSerial completes a TLS handshake against addr and returns the serial number
9675
// of the leaf certificate the server actually presented.
9776
func servedSerial(t *testing.T, addr string) int64 {
@@ -158,88 +137,3 @@ func TestReloadingTLSConfigServesRenewedCertificate(t *testing.T) {
158137
t.Fatalf("after renewal: expected renewed serial 2, got %d (certificate was not reloaded)", got)
159138
}
160139
}
161-
162-
// TestCertReloaderKeepsLastGoodCertificate verifies that a failed reload (e.g. a torn
163-
// read while the volume is updated) does not take the webhook down: it keeps serving
164-
// the previously loaded certificate instead of erroring the handshake.
165-
func TestCertReloaderKeepsLastGoodCertificate(t *testing.T) {
166-
dir := t.TempDir()
167-
certFile := filepath.Join(dir, "tls.crt")
168-
keyFile := filepath.Join(dir, "tls.key")
169-
170-
certA, keyA := genSelfSigned(t, 1)
171-
writeKeyPair(t, certFile, keyFile, certA, keyA, time.Now().Add(-2*time.Second))
172-
173-
cr, err := newCertReloader(certFile, keyFile)
174-
if err != nil {
175-
t.Fatalf("newCertReloader: %v", err)
176-
}
177-
178-
// Corrupt the cert file and advance its mtime so a reload is attempted and fails.
179-
if err := os.WriteFile(certFile, []byte("not a certificate"), 0o600); err != nil {
180-
t.Fatalf("corrupt cert: %v", err)
181-
}
182-
setModTime(t, certFile, time.Now().Add(2*time.Second))
183-
184-
got, err := cr.GetCertificate(&tls.ClientHelloInfo{})
185-
if err != nil {
186-
t.Fatalf("GetCertificate returned error instead of serving last good cert: %v", err)
187-
}
188-
if certSerial(t, got) != 1 {
189-
t.Fatalf("expected to keep serving serial 1 after failed reload, got %d", certSerial(t, got))
190-
}
191-
}
192-
193-
// TestCertReloaderRetriesAfterTransientFailure is the regression test for the reloader
194-
// abandoning a renewal on a transient error. A failed reload must NOT advance the
195-
// recorded mtime, so once the (unchanged-mtime) file becomes readable the renewed
196-
// certificate is still picked up. Against the code that advanced the mtime on failure,
197-
// the renewal is permanently missed and this test fails.
198-
func TestCertReloaderRetriesAfterTransientFailure(t *testing.T) {
199-
dir := t.TempDir()
200-
certFile := filepath.Join(dir, "tls.crt")
201-
keyFile := filepath.Join(dir, "tls.key")
202-
203-
certA, keyA := genSelfSigned(t, 1)
204-
writeKeyPair(t, certFile, keyFile, certA, keyA, time.Now().Add(-2*time.Second))
205-
206-
cr, err := newCertReloader(certFile, keyFile)
207-
if err != nil {
208-
t.Fatalf("newCertReloader: %v", err)
209-
}
210-
cr.retryInterval = 0 // retry immediately, no backoff wait in the test
211-
212-
// Renewal advances the mtime, but the first load fails (simulating a transient,
213-
// content-independent error) by writing an unreadable cert at the new mtime.
214-
renewMod := time.Now().Add(2 * time.Second)
215-
if err := os.WriteFile(certFile, []byte("transiently unreadable"), 0o600); err != nil {
216-
t.Fatalf("write bad cert: %v", err)
217-
}
218-
setModTime(t, certFile, renewMod)
219-
220-
if got, gerr := cr.GetCertificate(&tls.ClientHelloInfo{}); gerr != nil {
221-
t.Fatalf("GetCertificate errored during transient failure: %v", gerr)
222-
} else if certSerial(t, got) != 1 {
223-
t.Fatalf("during transient failure: expected to keep serial 1, got %d", certSerial(t, got))
224-
}
225-
226-
// The renewed, valid certificate is now readable at the SAME mtime as the failed
227-
// attempt. The reloader must still pick it up on retry.
228-
certB, keyB := genSelfSigned(t, 2)
229-
if err := os.WriteFile(certFile, certB, 0o600); err != nil {
230-
t.Fatalf("write renewed cert: %v", err)
231-
}
232-
if err := os.WriteFile(keyFile, keyB, 0o600); err != nil {
233-
t.Fatalf("write renewed key: %v", err)
234-
}
235-
setModTime(t, certFile, renewMod)
236-
setModTime(t, keyFile, renewMod)
237-
238-
got, gerr := cr.GetCertificate(&tls.ClientHelloInfo{})
239-
if gerr != nil {
240-
t.Fatalf("GetCertificate errored after retry: %v", gerr)
241-
}
242-
if certSerial(t, got) != 2 {
243-
t.Fatalf("after transient failure: expected renewed serial 2 on retry, got %d (mtime advanced on failure, renewal permanently missed)", certSerial(t, got))
244-
}
245-
}

0 commit comments

Comments
 (0)