Skip to content

Commit 44d9f53

Browse files
committed
fix(kubeovn-webhook): reload serving certificate on cert-manager renewal
The webhook loaded its TLS key pair once at startup and served it as a static tls.Config.Certificates entry for the lifetime of the process. A cert-manager renewal of the backing Secret was never picked up by a running pod, so once the certificate expired (about a year after install) the pod kept presenting the expired certificate. The kube-apiserver's TLS call to the webhook then failed and, because the MutatingWebhookConfiguration uses failurePolicy: Fail, every pod creation in tenant namespaces was rejected (including virt-launcher pods, blocking all VMIs). Serve the certificate through a reloading tls.Config.GetCertificate callback that re-reads the key pair when the mounted files change and keeps serving the last good certificate on a failed reload. Add a regression test that fails against the static implementation and passes once the certificate is reloaded. Signed-off-by: IvanHunters <[email protected]>
1 parent ee26a50 commit 44d9f53

3 files changed

Lines changed: 282 additions & 7 deletions

File tree

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
package main
2+
3+
import (
4+
"crypto/tls"
5+
"fmt"
6+
"log"
7+
"os"
8+
"sync"
9+
"time"
10+
)
11+
12+
// certReloader keeps the webhook serving certificate in sync with the key pair
13+
// files written by cert-manager.
14+
//
15+
// The certificate is served through tls.Config.GetCertificate, so a cert-manager
16+
// renewal of the backing Secret is picked up by the running process without a
17+
// restart. Without this, the key pair is read exactly once at startup and cached
18+
// for the lifetime of the process: after the certificate expires (roughly one year
19+
// after install) the pod keeps presenting the expired certificate, the
20+
// kube-apiserver's TLS call to the webhook fails, and because the
21+
// MutatingWebhookConfiguration uses failurePolicy: Fail, every pod creation in
22+
// tenant namespaces is rejected.
23+
type certReloader struct {
24+
certFile string
25+
keyFile string
26+
27+
mu sync.RWMutex
28+
cert *tls.Certificate
29+
modTime time.Time
30+
}
31+
32+
// newCertReloader loads the initial key pair and returns a reloader for it.
33+
func newCertReloader(certFile, keyFile string) (*certReloader, error) {
34+
cr := &certReloader{certFile: certFile, keyFile: keyFile}
35+
if err := cr.reload(); err != nil {
36+
return nil, err
37+
}
38+
return cr, nil
39+
}
40+
41+
// reload reads the key pair from disk and atomically swaps the cached certificate.
42+
func (cr *certReloader) reload() error {
43+
cert, err := tls.LoadX509KeyPair(cr.certFile, cr.keyFile)
44+
if err != nil {
45+
return err
46+
}
47+
48+
var modTime time.Time
49+
if fi, statErr := os.Stat(cr.certFile); statErr == nil {
50+
modTime = fi.ModTime()
51+
}
52+
53+
cr.mu.Lock()
54+
cr.cert = &cert
55+
cr.modTime = modTime
56+
cr.mu.Unlock()
57+
return nil
58+
}
59+
60+
// changed reports whether the certificate file was modified since the last load.
61+
func (cr *certReloader) changed() bool {
62+
fi, err := os.Stat(cr.certFile)
63+
if err != nil {
64+
return false
65+
}
66+
cr.mu.RLock()
67+
defer cr.mu.RUnlock()
68+
return fi.ModTime().After(cr.modTime)
69+
}
70+
71+
// GetCertificate is a tls.Config.GetCertificate callback. It reloads the key pair
72+
// when the file changes and keeps serving the last good certificate if a reload
73+
// fails (for example a torn read while the mounted Secret is being updated).
74+
func (cr *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
75+
if cr.changed() {
76+
if err := cr.reload(); err != nil {
77+
log.Printf("certificate reload failed, keeping previous certificate: %v", err)
78+
}
79+
}
80+
81+
cr.mu.RLock()
82+
defer cr.mu.RUnlock()
83+
if cr.cert == nil {
84+
return nil, fmt.Errorf("no certificate loaded")
85+
}
86+
return cr.cert, nil
87+
}
88+
89+
// newReloadingTLSConfig builds a tls.Config that serves the key pair via a
90+
// certReloader, so cert-manager renewals are honoured without a restart.
91+
func newReloadingTLSConfig(certFile, keyFile string) (*tls.Config, error) {
92+
cr, err := newCertReloader(certFile, keyFile)
93+
if err != nil {
94+
return nil, err
95+
}
96+
return &tls.Config{GetCertificate: cr.GetCertificate}, nil
97+
}
Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
package main
2+
3+
import (
4+
"crypto/ecdsa"
5+
"crypto/elliptic"
6+
"crypto/rand"
7+
"crypto/tls"
8+
"crypto/x509"
9+
"crypto/x509/pkix"
10+
"encoding/pem"
11+
"math/big"
12+
"os"
13+
"path/filepath"
14+
"testing"
15+
"time"
16+
)
17+
18+
// genSelfSigned returns a PEM-encoded self-signed cert/key pair carrying the given
19+
// serial number, so tests can tell two generations of the certificate apart.
20+
func genSelfSigned(t *testing.T, serial int64) (certPEM, keyPEM []byte) {
21+
t.Helper()
22+
23+
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
24+
if err != nil {
25+
t.Fatalf("generate key: %v", err)
26+
}
27+
28+
tmpl := &x509.Certificate{
29+
SerialNumber: big.NewInt(serial),
30+
Subject: pkix.Name{CommonName: "kube-ovn-webhook-test"},
31+
NotBefore: time.Now().Add(-time.Hour),
32+
NotAfter: time.Now().Add(time.Hour),
33+
KeyUsage: x509.KeyUsageDigitalSignature,
34+
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
35+
DNSNames: []string{"localhost"},
36+
BasicConstraintsValid: true,
37+
}
38+
39+
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
40+
if err != nil {
41+
t.Fatalf("create certificate: %v", err)
42+
}
43+
keyDER, err := x509.MarshalPKCS8PrivateKey(key)
44+
if err != nil {
45+
t.Fatalf("marshal key: %v", err)
46+
}
47+
48+
certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
49+
keyPEM = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER})
50+
return certPEM, keyPEM
51+
}
52+
53+
// writeKeyPair writes the cert/key files and bumps their mtime strictly forward,
54+
// mirroring how cert-manager replaces the mounted Secret on renewal.
55+
func writeKeyPair(t *testing.T, certFile, keyFile string, certPEM, keyPEM []byte, mtime time.Time) {
56+
t.Helper()
57+
if err := os.WriteFile(certFile, certPEM, 0o600); err != nil {
58+
t.Fatalf("write cert: %v", err)
59+
}
60+
if err := os.WriteFile(keyFile, keyPEM, 0o600); err != nil {
61+
t.Fatalf("write key: %v", err)
62+
}
63+
if err := os.Chtimes(certFile, mtime, mtime); err != nil {
64+
t.Fatalf("chtimes cert: %v", err)
65+
}
66+
if err := os.Chtimes(keyFile, mtime, mtime); err != nil {
67+
t.Fatalf("chtimes key: %v", err)
68+
}
69+
}
70+
71+
// servedSerial completes a TLS handshake against addr and returns the serial number
72+
// of the leaf certificate the server actually presented.
73+
func servedSerial(t *testing.T, addr string) int64 {
74+
t.Helper()
75+
conn, err := tls.Dial("tcp", addr, &tls.Config{InsecureSkipVerify: true}) //nolint:gosec // test-only, inspecting the served cert
76+
if err != nil {
77+
t.Fatalf("dial: %v", err)
78+
}
79+
defer conn.Close()
80+
certs := conn.ConnectionState().PeerCertificates
81+
if len(certs) == 0 {
82+
t.Fatalf("server presented no certificate")
83+
}
84+
return certs[0].SerialNumber.Int64()
85+
}
86+
87+
// TestReloadingTLSConfigServesRenewedCertificate is the regression test for the
88+
// cert-manager renewal not being picked up: it fails against the old code that
89+
// loads the key pair once into tls.Config.Certificates, and passes once the
90+
// certificate is served via a reloading GetCertificate callback.
91+
func TestReloadingTLSConfigServesRenewedCertificate(t *testing.T) {
92+
dir := t.TempDir()
93+
certFile := filepath.Join(dir, "tls.crt")
94+
keyFile := filepath.Join(dir, "tls.key")
95+
96+
certA, keyA := genSelfSigned(t, 1)
97+
writeKeyPair(t, certFile, keyFile, certA, keyA, time.Now().Add(-2*time.Second))
98+
99+
tlsConfig, err := newReloadingTLSConfig(certFile, keyFile)
100+
if err != nil {
101+
t.Fatalf("newReloadingTLSConfig: %v", err)
102+
}
103+
104+
ln, err := tls.Listen("tcp", "127.0.0.1:0", tlsConfig)
105+
if err != nil {
106+
t.Fatalf("listen: %v", err)
107+
}
108+
defer ln.Close()
109+
go func() {
110+
for {
111+
conn, err := ln.Accept()
112+
if err != nil {
113+
return
114+
}
115+
go func() {
116+
if tc, ok := conn.(*tls.Conn); ok {
117+
_ = tc.Handshake()
118+
}
119+
conn.Close()
120+
}()
121+
}
122+
}()
123+
124+
addr := ln.Addr().String()
125+
126+
if got := servedSerial(t, addr); got != 1 {
127+
t.Fatalf("before renewal: expected serial 1, got %d", got)
128+
}
129+
130+
// cert-manager renews the Secret: the mounted files are replaced in place.
131+
certB, keyB := genSelfSigned(t, 2)
132+
writeKeyPair(t, certFile, keyFile, certB, keyB, time.Now().Add(2*time.Second))
133+
134+
if got := servedSerial(t, addr); got != 2 {
135+
t.Fatalf("after renewal: expected renewed serial 2, got %d (certificate was not reloaded)", got)
136+
}
137+
}
138+
139+
// TestCertReloaderKeepsLastGoodCertificate verifies that a failed reload (e.g. a
140+
// torn read while the volume is updated) does not take the webhook down: it keeps
141+
// serving the previously loaded certificate instead of erroring the handshake.
142+
func TestCertReloaderKeepsLastGoodCertificate(t *testing.T) {
143+
dir := t.TempDir()
144+
certFile := filepath.Join(dir, "tls.crt")
145+
keyFile := filepath.Join(dir, "tls.key")
146+
147+
certA, keyA := genSelfSigned(t, 1)
148+
writeKeyPair(t, certFile, keyFile, certA, keyA, time.Now().Add(-2*time.Second))
149+
150+
cr, err := newCertReloader(certFile, keyFile)
151+
if err != nil {
152+
t.Fatalf("newCertReloader: %v", err)
153+
}
154+
155+
// Corrupt the cert file and advance its mtime so a reload is attempted and fails.
156+
if err := os.WriteFile(certFile, []byte("not a certificate"), 0o600); err != nil {
157+
t.Fatalf("corrupt cert: %v", err)
158+
}
159+
if err := os.Chtimes(certFile, time.Now().Add(2*time.Second), time.Now().Add(2*time.Second)); err != nil {
160+
t.Fatalf("chtimes: %v", err)
161+
}
162+
163+
got, err := cr.GetCertificate(&tls.ClientHelloInfo{})
164+
if err != nil {
165+
t.Fatalf("GetCertificate returned error instead of serving last good cert: %v", err)
166+
}
167+
if got == nil || got.Leaf == nil {
168+
// Leaf may be nil depending on the Go version; fall back to parsing.
169+
if got == nil || len(got.Certificate) == 0 {
170+
t.Fatalf("GetCertificate returned no certificate")
171+
}
172+
leaf, perr := x509.ParseCertificate(got.Certificate[0])
173+
if perr != nil {
174+
t.Fatalf("parse served cert: %v", perr)
175+
}
176+
got.Leaf = leaf
177+
}
178+
if got.Leaf.SerialNumber.Int64() != 1 {
179+
t.Fatalf("expected to keep serving serial 1 after failed reload, got %d", got.Leaf.SerialNumber.Int64())
180+
}
181+
}

‎packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package main
22

33
import (
4-
"crypto/tls"
54
"flag"
65
"log"
76
"net/http"
@@ -28,17 +27,15 @@ func main() {
2827
mux := http.NewServeMux()
2928
mux.HandleFunc("/mutate-pods", HandleMutatePods)
3029

31-
tlsCert, err := tls.LoadX509KeyPair(tlsCertFile, tlsKeyFile)
30+
tlsConfig, err := newReloadingTLSConfig(tlsCertFile, tlsKeyFile)
3231
if err != nil {
3332
log.Fatalf("Failed to load key pair: %v", err)
3433
}
3534

3635
server := &http.Server{
37-
Addr: ":8443",
38-
TLSConfig: &tls.Config{
39-
Certificates: []tls.Certificate{tlsCert},
40-
},
41-
Handler: mux,
36+
Addr: ":8443",
37+
TLSConfig: tlsConfig,
38+
Handler: mux,
4239
}
4340

4441
log.Printf("Starting webhook server on %s", server.Addr)

0 commit comments

Comments
 (0)