Skip to content

Commit cd89f94

Browse files
author
Andrew Lytvynov
committed
Add TLS support to exec authenticator plugin
https://github.com/kubernetes/community/blob/master/contributors/design-proposals/auth/kubectl-exec-plugins.md#tls-client-certificate-support Allows exec plugin to return raw TLS key/cert data. This data populates transport.Config.TLS fields. transport.Config.TLS propagates custom credentials using tls.Config.GetClientCertificate callback. On key/cert rotation, all connections using old credentials are closed
1 parent f3d54f3 commit cd89f94

File tree

19 files changed

+633
-111
lines changed

19 files changed

+633
-111
lines changed

‎staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json

+4
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎staging/src/k8s.io/apiserver/Godeps/Godeps.json

+4
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎staging/src/k8s.io/client-go/pkg/apis/clientauthentication/types.go

+7
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,14 @@ type ExecCredentialStatus struct {
5757
// +optional
5858
ExpirationTimestamp *metav1.Time
5959
// Token is a bearer token used by the client for request authentication.
60+
// +optional
6061
Token string
62+
// PEM-encoded client TLS certificate.
63+
// +optional
64+
ClientCertificateData string
65+
// PEM-encoded client TLS private key.
66+
// +optional
67+
ClientKeyData string
6168
}
6269

6370
// Response defines metadata about a failed request, including HTTP status code and

‎staging/src/k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1/types.go

+8
Original file line numberDiff line numberDiff line change
@@ -52,12 +52,20 @@ type ExecCredentialSpec struct {
5252
}
5353

5454
// ExecCredentialStatus holds credentials for the transport to use.
55+
//
56+
// Token and ClientKeyData are sensitive fields. This data should only be
57+
// transmitted in-memory between client and exec plugin process. Exec plugin
58+
// itself should at least be protected via file permissions.
5559
type ExecCredentialStatus struct {
5660
// ExpirationTimestamp indicates a time when the provided credentials expire.
5761
// +optional
5862
ExpirationTimestamp *metav1.Time `json:"expirationTimestamp,omitempty"`
5963
// Token is a bearer token used by the client for request authentication.
6064
Token string `json:"token,omitempty"`
65+
// PEM-encoded client TLS certificates (including intermediates, if any).
66+
ClientCertificateData string `json:"clientCertificateData,omitempty"`
67+
// PEM-encoded private key for the above certificate.
68+
ClientKeyData string `json:"clientKeyData,omitempty"`
6169
}
6270

6371
// Response defines metadata about a failed request, including HTTP status code and

‎staging/src/k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1/zz_generated.conversion.go

+4
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/BUILD

+5
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ go_library(
1515
"//vendor/k8s.io/client-go/pkg/apis/clientauthentication:go_default_library",
1616
"//vendor/k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1:go_default_library",
1717
"//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
18+
"//vendor/k8s.io/client-go/transport:go_default_library",
19+
"//vendor/k8s.io/client-go/util/connrotation:go_default_library",
1820
],
1921
)
2022

@@ -24,8 +26,11 @@ go_test(
2426
data = glob(["testdata/**"]),
2527
embed = [":go_default_library"],
2628
deps = [
29+
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
30+
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
2731
"//vendor/k8s.io/client-go/pkg/apis/clientauthentication:go_default_library",
2832
"//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
33+
"//vendor/k8s.io/client-go/transport:go_default_library",
2934
],
3035
)
3136

‎staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/exec.go

+115-33
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,15 @@ package exec
1818

1919
import (
2020
"bytes"
21+
"context"
22+
"crypto/tls"
2123
"fmt"
2224
"io"
25+
"net"
2326
"net/http"
2427
"os"
2528
"os/exec"
29+
"reflect"
2630
"sync"
2731
"time"
2832

@@ -35,6 +39,8 @@ import (
3539
"k8s.io/client-go/pkg/apis/clientauthentication"
3640
"k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1"
3741
"k8s.io/client-go/tools/clientcmd/api"
42+
"k8s.io/client-go/transport"
43+
"k8s.io/client-go/util/connrotation"
3844
)
3945

4046
const execInfoEnv = "KUBERNETES_EXEC_INFO"
@@ -147,14 +153,55 @@ type Authenticator struct {
147153
// The mutex also guards calling the plugin. Since the plugin could be
148154
// interactive we want to make sure it's only called once.
149155
mu sync.Mutex
150-
cachedToken string
156+
cachedCreds *credentials
151157
exp time.Time
158+
159+
onRotate func()
152160
}
153161

154-
// WrapTransport instruments an existing http.RoundTripper with credentials returned
155-
// by the plugin.
156-
func (a *Authenticator) WrapTransport(rt http.RoundTripper) http.RoundTripper {
157-
return &roundTripper{a, rt}
162+
type credentials struct {
163+
token string
164+
cert *tls.Certificate
165+
}
166+
167+
// UpdateTransportConfig updates the transport.Config to use credentials
168+
// returned by the plugin.
169+
func (a *Authenticator) UpdateTransportConfig(c *transport.Config) error {
170+
wt := c.WrapTransport
171+
c.WrapTransport = func(rt http.RoundTripper) http.RoundTripper {
172+
if wt != nil {
173+
rt = wt(rt)
174+
}
175+
return &roundTripper{a, rt}
176+
}
177+
178+
getCert := c.TLS.GetCert
179+
c.TLS.GetCert = func() (*tls.Certificate, error) {
180+
// If previous GetCert is present and returns a valid non-nil
181+
// certificate, use that. Otherwise use cert from exec plugin.
182+
if getCert != nil {
183+
cert, err := getCert()
184+
if err != nil {
185+
return nil, err
186+
}
187+
if cert != nil {
188+
return cert, nil
189+
}
190+
}
191+
return a.cert()
192+
}
193+
194+
var dial func(ctx context.Context, network, addr string) (net.Conn, error)
195+
if c.Dial != nil {
196+
dial = c.Dial
197+
} else {
198+
dial = (&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext
199+
}
200+
d := connrotation.NewDialer(dial)
201+
a.onRotate = d.CloseAll
202+
c.Dial = d.DialContext
203+
204+
return nil
158205
}
159206

160207
type roundTripper struct {
@@ -169,11 +216,13 @@ func (r *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
169216
return r.base.RoundTrip(req)
170217
}
171218

172-
token, err := r.a.token()
219+
creds, err := r.a.getCreds()
173220
if err != nil {
174-
return nil, fmt.Errorf("getting token: %v", err)
221+
return nil, fmt.Errorf("getting credentials: %v", err)
222+
}
223+
if creds.token != "" {
224+
req.Header.Set("Authorization", "Bearer "+creds.token)
175225
}
176-
req.Header.Set("Authorization", "Bearer "+token)
177226

178227
res, err := r.base.RoundTrip(req)
179228
if err != nil {
@@ -184,47 +233,60 @@ func (r *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
184233
Header: res.Header,
185234
Code: int32(res.StatusCode),
186235
}
187-
if err := r.a.refresh(token, resp); err != nil {
188-
glog.Errorf("refreshing token: %v", err)
236+
if err := r.a.maybeRefreshCreds(creds, resp); err != nil {
237+
glog.Errorf("refreshing credentials: %v", err)
189238
}
190239
}
191240
return res, nil
192241
}
193242

194-
func (a *Authenticator) tokenExpired() bool {
243+
func (a *Authenticator) credsExpired() bool {
195244
if a.exp.IsZero() {
196245
return false
197246
}
198247
return a.now().After(a.exp)
199248
}
200249

201-
func (a *Authenticator) token() (string, error) {
250+
func (a *Authenticator) cert() (*tls.Certificate, error) {
251+
creds, err := a.getCreds()
252+
if err != nil {
253+
return nil, err
254+
}
255+
return creds.cert, nil
256+
}
257+
258+
func (a *Authenticator) getCreds() (*credentials, error) {
202259
a.mu.Lock()
203260
defer a.mu.Unlock()
204-
if a.cachedToken != "" && !a.tokenExpired() {
205-
return a.cachedToken, nil
261+
if a.cachedCreds != nil && !a.credsExpired() {
262+
return a.cachedCreds, nil
206263
}
207264

208-
return a.getToken(nil)
265+
if err := a.refreshCredsLocked(nil); err != nil {
266+
return nil, err
267+
}
268+
return a.cachedCreds, nil
209269
}
210270

211-
// refresh executes the plugin to force a rotation of the token.
212-
func (a *Authenticator) refresh(token string, r *clientauthentication.Response) error {
271+
// maybeRefreshCreds executes the plugin to force a rotation of the
272+
// credentials, unless they were rotated already.
273+
func (a *Authenticator) maybeRefreshCreds(creds *credentials, r *clientauthentication.Response) error {
213274
a.mu.Lock()
214275
defer a.mu.Unlock()
215276

216-
if token != a.cachedToken {
217-
// Token already rotated.
277+
// Since we're not making a new pointer to a.cachedCreds in getCreds, no
278+
// need to do deep comparison.
279+
if creds != a.cachedCreds {
280+
// Credentials already rotated.
218281
return nil
219282
}
220283

221-
_, err := a.getToken(r)
222-
return err
284+
return a.refreshCredsLocked(r)
223285
}
224286

225-
// getToken executes the plugin and reads the credentials from stdout. It must be
226-
// called while holding the Authenticator's mutex.
227-
func (a *Authenticator) getToken(r *clientauthentication.Response) (string, error) {
287+
// refreshCredsLocked executes the plugin and reads the credentials from
288+
// stdout. It must be called while holding the Authenticator's mutex.
289+
func (a *Authenticator) refreshCredsLocked(r *clientauthentication.Response) error {
228290
cred := &clientauthentication.ExecCredential{
229291
Spec: clientauthentication.ExecCredentialSpec{
230292
Response: r,
@@ -234,7 +296,7 @@ func (a *Authenticator) getToken(r *clientauthentication.Response) (string, erro
234296

235297
data, err := runtime.Encode(codecs.LegacyCodec(a.group), cred)
236298
if err != nil {
237-
return "", fmt.Errorf("encode ExecCredentials: %v", err)
299+
return fmt.Errorf("encode ExecCredentials: %v", err)
238300
}
239301

240302
env := append(a.environ(), a.env...)
@@ -250,31 +312,51 @@ func (a *Authenticator) getToken(r *clientauthentication.Response) (string, erro
250312
}
251313

252314
if err := cmd.Run(); err != nil {
253-
return "", fmt.Errorf("exec: %v", err)
315+
return fmt.Errorf("exec: %v", err)
254316
}
255317

256318
_, gvk, err := codecs.UniversalDecoder(a.group).Decode(stdout.Bytes(), nil, cred)
257319
if err != nil {
258-
return "", fmt.Errorf("decode stdout: %v", err)
320+
return fmt.Errorf("decoding stdout: %v", err)
259321
}
260322
if gvk.Group != a.group.Group || gvk.Version != a.group.Version {
261-
return "", fmt.Errorf("exec plugin is configured to use API version %s, plugin returned version %s",
323+
return fmt.Errorf("exec plugin is configured to use API version %s, plugin returned version %s",
262324
a.group, schema.GroupVersion{Group: gvk.Group, Version: gvk.Version})
263325
}
264326

265327
if cred.Status == nil {
266-
return "", fmt.Errorf("exec plugin didn't return a status field")
328+
return fmt.Errorf("exec plugin didn't return a status field")
267329
}
268-
if cred.Status.Token == "" {
269-
return "", fmt.Errorf("exec plugin didn't return a token")
330+
if cred.Status.Token == "" && cred.Status.ClientCertificateData == "" && cred.Status.ClientKeyData == "" {
331+
return fmt.Errorf("exec plugin didn't return a token or cert/key pair")
332+
}
333+
if (cred.Status.ClientCertificateData == "") != (cred.Status.ClientKeyData == "") {
334+
return fmt.Errorf("exec plugin returned only certificate or key, not both")
270335
}
271336

272337
if cred.Status.ExpirationTimestamp != nil {
273338
a.exp = cred.Status.ExpirationTimestamp.Time
274339
} else {
275340
a.exp = time.Time{}
276341
}
277-
a.cachedToken = cred.Status.Token
278342

279-
return a.cachedToken, nil
343+
newCreds := &credentials{
344+
token: cred.Status.Token,
345+
}
346+
if cred.Status.ClientKeyData != "" && cred.Status.ClientCertificateData != "" {
347+
cert, err := tls.X509KeyPair([]byte(cred.Status.ClientCertificateData), []byte(cred.Status.ClientKeyData))
348+
if err != nil {
349+
return fmt.Errorf("failed parsing client key/certificate: %v", err)
350+
}
351+
newCreds.cert = &cert
352+
}
353+
354+
oldCreds := a.cachedCreds
355+
a.cachedCreds = newCreds
356+
// Only close all connections when TLS cert rotates. Token rotation doesn't
357+
// need the extra noise.
358+
if a.onRotate != nil && oldCreds != nil && !reflect.DeepEqual(oldCreds.cert, a.cachedCreds.cert) {
359+
a.onRotate()
360+
}
361+
return nil
280362
}

0 commit comments

Comments
 (0)