|
1 | 1 | package main |
2 | 2 |
|
3 | | -import ( |
4 | | - "crypto/tls" |
5 | | - "fmt" |
6 | | - "log" |
7 | | - "os" |
8 | | - "sync" |
9 | | - "time" |
10 | | -) |
| 3 | +import "crypto/tls" |
11 | 4 |
|
12 | | -// certReloader keeps the webhook serving certificate in sync with the key pair |
13 | | -// files written by cert-manager. |
| 5 | +// newReloadingTLSConfig returns a tls.Config that reloads the certificate and |
| 6 | +// key from disk on every TLS handshake, so a cert-manager renewal of the mounted |
| 7 | +// Secret is picked up by the running process without a restart. |
14 | 8 | // |
15 | | -// The certificate is served through tls.Config.GetCertificate, so a cert-manager |
16 | | -// renewal of the backing Secret is picked up by the running process without a |
17 | | -// restart. Without this, the key pair is read exactly once at startup and cached |
18 | | -// for the lifetime of the process: after the certificate expires (roughly one year |
19 | | -// after install) the pod keeps presenting the expired certificate, the |
20 | | -// kube-apiserver's TLS call to the webhook fails, and because the |
21 | | -// MutatingWebhookConfiguration uses failurePolicy: Fail, every pod creation in |
22 | | -// tenant namespaces is rejected. |
23 | | -type certReloader struct { |
24 | | - certFile string |
25 | | - keyFile string |
26 | | - |
27 | | - // retryInterval bounds how often a failed reload (or a repeated stat failure) is |
28 | | - // retried, so a persistently broken file does not read/parse/log on every handshake. |
29 | | - retryInterval time.Duration |
30 | | - |
31 | | - mu sync.RWMutex |
32 | | - cert *tls.Certificate |
33 | | - loadedModTime time.Time |
34 | | - reloading bool |
35 | | - nextRetry time.Time |
36 | | - nextLog time.Time |
37 | | -} |
38 | | - |
39 | | -const defaultReloadRetryInterval = 30 * time.Second |
40 | | - |
41 | | -// newCertReloader loads the initial key pair and returns a reloader for it. |
42 | | -func newCertReloader(certFile, keyFile string) (*certReloader, error) { |
43 | | - cr := &certReloader{ |
44 | | - certFile: certFile, |
45 | | - keyFile: keyFile, |
46 | | - retryInterval: defaultReloadRetryInterval, |
47 | | - } |
48 | | - modTime, err := fileModTime(certFile) |
49 | | - if err != nil { |
50 | | - return nil, err |
51 | | - } |
52 | | - if err := cr.load(modTime); err != nil { |
53 | | - return nil, err |
54 | | - } |
55 | | - return cr, nil |
56 | | -} |
57 | | - |
58 | | -// fileModTime returns the modification time of the given file. |
59 | | -func fileModTime(name string) (time.Time, error) { |
60 | | - fi, err := os.Stat(name) |
61 | | - if err != nil { |
62 | | - return time.Time{}, err |
63 | | - } |
64 | | - return fi.ModTime(), nil |
65 | | -} |
66 | | - |
67 | | -// load reads the key pair and, on success, caches it tagged with modTime. On failure |
68 | | -// the cached certificate and its recorded mtime are left unchanged, so a transient |
69 | | -// failure on a valid file is retried rather than the renewal being abandoned. |
70 | | -func (cr *certReloader) load(modTime time.Time) error { |
71 | | - cert, err := tls.LoadX509KeyPair(cr.certFile, cr.keyFile) |
72 | | - if err != nil { |
73 | | - return err |
74 | | - } |
75 | | - cr.mu.Lock() |
76 | | - cr.cert = &cert |
77 | | - cr.loadedModTime = modTime |
78 | | - cr.mu.Unlock() |
79 | | - return nil |
80 | | -} |
81 | | - |
82 | | -// refresh reloads the key pair when the certificate file's modification time changes. |
| 9 | +// Without this, the key pair is read exactly once at startup and cached for the |
| 10 | +// lifetime of the process: after the certificate expires (roughly one year after |
| 11 | +// install) the pod keeps presenting the expired certificate, the kube-apiserver's |
| 12 | +// TLS call to the webhook fails, and because the MutatingWebhookConfiguration uses |
| 13 | +// failurePolicy: Fail, every pod creation in tenant namespaces is rejected. |
83 | 14 | // |
84 | | -// A failed reload keeps the last good certificate and does NOT advance the recorded |
85 | | -// mtime, so a transient error (fd exhaustion, a torn read) is retried, at most once per |
86 | | -// retryInterval, until it succeeds: a transient failure must never permanently pin an |
87 | | -// expiring certificate. A single reload is kept in flight so a burst of handshakes does |
88 | | -// not trigger a thundering herd of reads. |
89 | | -func (cr *certReloader) refresh() { |
90 | | - modTime, err := fileModTime(cr.certFile) |
91 | | - if err != nil { |
92 | | - cr.logRateLimited("certificate stat failed, keeping previous certificate: %v", err) |
93 | | - return |
94 | | - } |
95 | | - |
96 | | - cr.mu.Lock() |
97 | | - if modTime.Equal(cr.loadedModTime) || cr.reloading || time.Now().Before(cr.nextRetry) { |
98 | | - cr.mu.Unlock() |
99 | | - return |
100 | | - } |
101 | | - cr.reloading = true |
102 | | - cr.mu.Unlock() |
103 | | - |
104 | | - loadErr := cr.load(modTime) |
105 | | - |
106 | | - cr.mu.Lock() |
107 | | - cr.reloading = false |
108 | | - if loadErr == nil { |
109 | | - cr.nextRetry = time.Time{} |
110 | | - } else { |
111 | | - cr.nextRetry = time.Now().Add(cr.retryInterval) |
112 | | - } |
113 | | - cr.mu.Unlock() |
114 | | - |
115 | | - if loadErr != nil { |
116 | | - cr.logRateLimited("certificate reload failed, keeping previous certificate: %v", loadErr) |
117 | | - } |
118 | | -} |
119 | | - |
120 | | -// logRateLimited emits a log line at most once per retryInterval, so a persistently |
121 | | -// broken mount does not spam the log on every TLS handshake. |
122 | | -func (cr *certReloader) logRateLimited(format string, args ...any) { |
123 | | - cr.mu.Lock() |
124 | | - if time.Now().Before(cr.nextLog) { |
125 | | - cr.mu.Unlock() |
126 | | - return |
127 | | - } |
128 | | - cr.nextLog = time.Now().Add(cr.retryInterval) |
129 | | - cr.mu.Unlock() |
130 | | - log.Printf(format, args...) |
131 | | -} |
132 | | - |
133 | | -// GetCertificate is a tls.Config.GetCertificate callback. It refreshes the key pair |
134 | | -// when the certificate file changes and keeps serving the last good certificate if a |
135 | | -// reload fails. |
136 | | -func (cr *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) { |
137 | | - cr.refresh() |
138 | | - |
139 | | - cr.mu.RLock() |
140 | | - defer cr.mu.RUnlock() |
141 | | - if cr.cert == nil { |
142 | | - return nil, fmt.Errorf("no certificate loaded") |
143 | | - } |
144 | | - return cr.cert, nil |
145 | | -} |
146 | | - |
147 | | -// newReloadingTLSConfig builds a tls.Config that serves the key pair via a |
148 | | -// certReloader, so cert-manager renewals are honoured without a restart. |
| 15 | +// tls.Config.GetCertificate is invoked on every handshake whenever Certificates is |
| 16 | +// left unset, so re-reading the files there is enough; no watcher, mtime tracking |
| 17 | +// or cache is needed: |
| 18 | +// - The cert/key files are mounted from a plain Secret volume with no subPath. |
| 19 | +// Kubernetes' atomic writer swaps the whole ..data directory via a single |
| 20 | +// symlink flip, so a reader always sees a complete old or complete new |
| 21 | +// generation of both files, never a torn mid-write mixture. A per-handshake |
| 22 | +// LoadX509KeyPair therefore cannot observe a partial renewal. |
| 23 | +// - The per-handshake cost (a few KB of file I/O plus a PEM/key parse) is |
| 24 | +// negligible next to the asymmetric crypto the handshake already performs. |
| 25 | +// - The webhook configures no mTLS (no ClientCAs), so GetCertificate's |
| 26 | +// limitation of not refreshing client CA pools does not apply. |
| 27 | +// |
| 28 | +// If the mounted files genuinely become unreadable (Secret deleted, permissions |
| 29 | +// broken: an operator error, not a normal renewal) the handshake fails loudly |
| 30 | +// rather than silently serving a stale certificate. |
149 | 31 | func newReloadingTLSConfig(certFile, keyFile string) (*tls.Config, error) { |
150 | | - cr, err := newCertReloader(certFile, keyFile) |
151 | | - if err != nil { |
| 32 | + // Fail fast at startup if the initial key pair is missing or malformed. |
| 33 | + if _, err := tls.LoadX509KeyPair(certFile, keyFile); err != nil { |
152 | 34 | return nil, err |
153 | 35 | } |
154 | | - return &tls.Config{GetCertificate: cr.GetCertificate}, nil |
| 36 | + return &tls.Config{ |
| 37 | + GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { |
| 38 | + cert, err := tls.LoadX509KeyPair(certFile, keyFile) |
| 39 | + if err != nil { |
| 40 | + return nil, err |
| 41 | + } |
| 42 | + return &cert, nil |
| 43 | + }, |
| 44 | + }, nil |
155 | 45 | } |
0 commit comments