Skip to content

Commit 92040fe

Browse files
authored
Verify actor JWTs in the credential exchange through OIDC discovery (#3447)
Part of #3095. The objectstore-plugin exchange read actor JWT keys from a file, a stopgap from before ate-idp-server. #3358 needs an issuer instead, so its `pkg/` API takes no key file ([thread](#3358 (comment))). - The exchange fetches keys through `<issuer>/.well-known/openid-configuration`. `--actor-jwt-jwks-file`, `oidcjwt.FileKeySource` and the `actor-id-jwt-jwks` volume are removed. - The sidecar verifies the issuer's certificate against the system roots plus `--actor-jwt-issuer-ca-file`, which is new and defaults to the service DNS trust bundle. The default ate-idp-server issuer verifies with or without an explicit `:443`, and a custom issuer can use a public certificate. ate-setup writes no CA setting for it. The sidecar re-reads the bundle when it changes and keeps the last good pool if a re-read fails. ate-api-server's `JWTProvider` CA files use the same reloading client and still replace the system roots there. - Nothing reads the pool Secret's `jwks` key now, so its writers are removed: the `admin` commands of kubectl-ate, ate-setup, `PublicJWKS` and `ConcretePool.JWKS`. Existing Secrets keep the stale key. - The sidecar fetches keys once at start. It exits on a certificate error and only warns on other errors, so an unreachable issuer fails exchanges with `UNAVAILABLE` until a fetch succeeds. Cached keys refresh after 5m, a failed refresh retries after 1m, and a token with an unknown key ID triggers at most one fetch per 1m. Until a fetch completes, including one the caller canceled, such a token gets `UNAVAILABLE`, so a newly rotated key ID is retried. Upgrade: drop `--actor-jwt-jwks-file` from custom sidecar configs. Set `--actor-jwt-issuer-ca-file` only if the issuer uses a private CA other than the service DNS one. Tests pass for `oidcjwt`, `credbundle`, `objectstoreplugin`, `cmd/objectstore-plugin`, `cmd/ate-setup` and `apiauthn`. The snapshotcredentials e2e needed no change. It compiles with `-tags=e2e` but has not run, as it needs GCS. - [x] Tests pass - [x] Docs updated --------- Signed-off-by: Anish Ramasekar <[email protected]>
1 parent a6442f9 commit 92040fe

25 files changed

Lines changed: 1010 additions & 346 deletions

File tree

‎cmd/ate-setup/internal/steps/create.go‎

Lines changed: 4 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
package steps
1616

1717
import (
18-
"bytes"
1918
"context"
2019
"fmt"
2120
"strings"
@@ -57,7 +56,7 @@ const (
5756
const caValidity = 365 * 24 * time.Hour
5857

5958
// CreateJWTAuthorityPoolSecret generates the actor-identity JWT signing pool.
60-
// An existing pool is kept, and its "jwks" key is brought in step with it.
59+
// An existing pool is kept.
6160
// This is the make-jwt-pool call create_jwt_authority_pool_secret shelled out
6261
// to kubectl-ate for.
6362
func (e *Env) CreateJWTAuthorityPoolSecret(ctx context.Context) error {
@@ -367,7 +366,7 @@ func (e *Env) createJWTPool(ctx context.Context, namespace, name string) error {
367366
}
368367
if existing != nil {
369368
log.Infof(" JWT authority pool %s/%s already exists; keeping it", namespace, name)
370-
return e.publishJWTPoolKeys(ctx, existing)
369+
return nil
371370
}
372371

373372
data, err := newJWTPoolSecretData(e.Cfg.ActorJWTAlgorithm)
@@ -377,40 +376,14 @@ func (e *Env) createJWTPool(ctx context.Context, namespace, name string) error {
377376
return e.createPoolSecret(ctx, namespace, name, corev1.SecretTypeOpaque, data)
378377
}
379378

380-
// publishJWTPoolKeys makes the "jwks" key of a pool Secret match its pool:
381-
// missing on a Secret from before the key existed, or stale after a tool that
382-
// did not know it rewrote the pool. A stale set would keep a removed key
383-
// trusted, so it is recomputed from the pool every run. A matching Secret is
384-
// left alone.
385-
func (e *Env) publishJWTPoolKeys(ctx context.Context, secret *corev1.Secret) error {
386-
jwks, err := localjwtauthority.PublicJWKS(secret.Data["pool"])
387-
if err != nil {
388-
return fmt.Errorf("while publishing the public keys of %s/%s: %w", secret.Namespace, secret.Name, err)
389-
}
390-
if bytes.Equal(secret.Data["jwks"], jwks) {
391-
return nil
392-
}
393-
secret.Data["jwks"] = jwks
394-
if _, err := e.Kube.Typed.CoreV1().Secrets(secret.Namespace).Update(ctx, secret, metav1.UpdateOptions{}); err != nil {
395-
return fmt.Errorf("while writing jwks to %s/%s: %w", secret.Namespace, secret.Name, err)
396-
}
397-
log.Infof(" published the public keys of %s/%s as jwks", secret.Namespace, secret.Name)
398-
return nil
399-
}
400-
401379
// newJWTPoolSecretData generates a pool with one active authority for
402-
// algorithm, keyed by its thumbprint. "jwks" holds the pool's public keys for
403-
// verifiers that must not see the pool.
380+
// algorithm, keyed by its thumbprint.
404381
func newJWTPoolSecretData(algorithm string) (map[string][]byte, error) {
405382
poolBytes, _, err := localjwtauthority.GeneratePool(algorithm, "")
406383
if err != nil {
407384
return nil, fmt.Errorf("while generating the JWT pool: %w", err)
408385
}
409-
jwks, err := localjwtauthority.PublicJWKS(poolBytes)
410-
if err != nil {
411-
return nil, fmt.Errorf("while publishing the JWT pool's public keys: %w", err)
412-
}
413-
return map[string][]byte{"pool": poolBytes, "jwks": jwks}, nil
386+
return map[string][]byte{"pool": poolBytes}, nil
414387
}
415388

416389
// createPoolSecret writes pool state.

‎cmd/ate-setup/internal/steps/create_test.go‎

Lines changed: 12 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -202,17 +202,14 @@ func TestNewJWTPoolSecretData(t *testing.T) {
202202
if err != nil {
203203
t.Fatalf("newJWTPoolSecretData() error = %v", err)
204204
}
205-
if diff := cmp.Diff([]string{"jwks", "pool"}, slices.Sorted(maps.Keys(data))); diff != "" {
205+
if diff := cmp.Diff([]string{"pool"}, slices.Sorted(maps.Keys(data))); diff != "" {
206206
t.Errorf("secret keys differ (-want +got):\n%s", diff)
207207
}
208208

209209
pool, err := localjwtauthority.Unmarshal(data["pool"])
210210
if err != nil {
211211
t.Fatalf("Unmarshal() error = %v", err)
212212
}
213-
if jwks, err := pool.JWKS(); err != nil || !bytes.Equal(jwks, data["jwks"]) {
214-
t.Errorf("jwks = %s, want the pool's public keys (%v)", data["jwks"], err)
215-
}
216213
if len(pool.Authorities) != 1 {
217214
t.Fatalf("pool has %d authorities, want 1", len(pool.Authorities))
218215
}
@@ -266,23 +263,19 @@ func TestEnsurePodCertificateCAs(t *testing.T) {
266263
}
267264
}
268265

269-
func TestCreateJWTPoolPublishesKeysOfExistingPool(t *testing.T) {
266+
// TestCreateJWTPoolKeepsExistingPool checks that an existing pool Secret is
267+
// left as it is, including a "jwks" key that earlier versions wrote.
268+
func TestCreateJWTPoolKeepsExistingPool(t *testing.T) {
270269
poolBytes, _, err := localjwtauthority.GeneratePool("ES256", "")
271270
if err != nil {
272271
t.Fatal(err)
273272
}
274-
wantJWKS, err := localjwtauthority.PublicJWKS(poolBytes)
275-
if err != nil {
276-
t.Fatal(err)
277-
}
278273
for _, tc := range []struct {
279-
name string
280-
data map[string][]byte
281-
wantUpdates int
274+
name string
275+
data map[string][]byte
282276
}{
283-
{"pool from before jwks", map[string][]byte{"pool": poolBytes}, 1},
284-
{"pool with a stale jwks", map[string][]byte{"pool": poolBytes, "jwks": []byte(`{"keys":[]}`)}, 1},
285-
{"pool with jwks", map[string][]byte{"pool": poolBytes, "jwks": wantJWKS}, 0},
277+
{"pool", map[string][]byte{"pool": poolBytes}},
278+
{"pool with a stale jwks", map[string][]byte{"pool": poolBytes, "jwks": []byte(`{"keys":[]}`)}},
286279
} {
287280
t.Run(tc.name, func(t *testing.T) {
288281
kc := fake.NewSimpleClientset(&corev1.Secret{
@@ -302,11 +295,11 @@ func TestCreateJWTPoolPublishesKeysOfExistingPool(t *testing.T) {
302295
if err != nil {
303296
t.Fatal(err)
304297
}
305-
if !bytes.Equal(got.Data["pool"], poolBytes) || !bytes.Equal(got.Data["jwks"], wantJWKS) {
306-
t.Errorf("secret data = %v keys, want the same pool and its jwks", slices.Sorted(maps.Keys(got.Data)))
298+
if diff := cmp.Diff(tc.data, got.Data); diff != "" {
299+
t.Errorf("secret data differs (-want +got):\n%s", diff)
307300
}
308-
if updates != tc.wantUpdates {
309-
t.Errorf("updates = %d, want %d", updates, tc.wantUpdates)
301+
if updates != 0 {
302+
t.Errorf("updates = %d, want 0", updates)
310303
}
311304
})
312305
}

‎cmd/ate-setup/internal/steps/prereqs.go‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,7 @@ var trustBundleNames = []string{
3939
func (e *Env) EnsureAPIServerPrerequisites(ctx context.Context) error {
4040
log.Step("ensure_apiserver_prerequisites")
4141

42-
// Not through ensureSecret: an existing pool Secret may still need its
43-
// public keys published, which CreateJWTAuthorityPoolSecret handles.
44-
if err := e.CreateJWTAuthorityPoolSecret(ctx); err != nil {
42+
if err := e.ensureSecret(ctx, e.Namespace(), SecretActorIDJWTPool, e.CreateJWTAuthorityPoolSecret); err != nil {
4543
return err
4644
}
4745
if err := e.ensureSecret(ctx, e.Namespace(), SecretActorIDCAPool, e.CreateActorIDCAPoolSecret); err != nil {

‎cmd/kubectl-ate/internal/cmd/admin.go‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -259,13 +259,7 @@ func updateJWTPool(ctx context.Context, errOut io.Writer, secrets typedcorev1.Se
259259
if err != nil {
260260
return fmt.Errorf("while marshaling pool: %w", err)
261261
}
262-
jwks, err := pool.JWKS()
263-
if err != nil {
264-
return fmt.Errorf("while publishing the pool's public keys: %w", err)
265-
}
266-
// Written together so a verifier reading "jwks" never lags "pool".
267262
secret.Data["pool"] = wire
268-
secret.Data["jwks"] = jwks
269263
if _, err := secrets.Update(ctx, secret, metav1.UpdateOptions{}); err != nil {
270264
return fmt.Errorf("while writing pool secret: %w", err)
271265
}
@@ -337,19 +331,13 @@ func newJWTPoolSecret(namespace, name, algorithm, keyID string) (*corev1.Secret,
337331
if err != nil {
338332
return nil, "", fmt.Errorf("while generating JWT authority pool: %w", err)
339333
}
340-
jwks, err := localjwtauthority.PublicJWKS(poolBytes)
341-
if err != nil {
342-
return nil, "", fmt.Errorf("while publishing the pool's public keys: %w", err)
343-
}
344-
345334
return &corev1.Secret{
346335
ObjectMeta: metav1.ObjectMeta{
347336
Namespace: namespace,
348337
Name: name,
349338
},
350339
Data: map[string][]byte{
351340
"pool": poolBytes,
352-
"jwks": jwks,
353341
},
354342
}, id, nil
355343
}

‎cmd/kubectl-ate/internal/cmd/admin_test.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,10 @@
1515
package cmd
1616

1717
import (
18-
"bytes"
1918
"context"
2019
"io"
20+
"maps"
21+
"slices"
2122
"strings"
2223
"testing"
2324

@@ -57,6 +58,9 @@ func TestNewJWTPoolSecretWithFlagDefaults(t *testing.T) {
5758
if secret.Namespace != "ate-system" || secret.Name != "actor-id-jwt-pool" {
5859
t.Errorf("secret = %s/%s, want ate-system/actor-id-jwt-pool", secret.Namespace, secret.Name)
5960
}
61+
if diff := cmp.Diff([]string{"pool"}, slices.Sorted(maps.Keys(secret.Data))); diff != "" {
62+
t.Errorf("secret keys differ (-want +got):\n%s", diff)
63+
}
6064
pool, err := localjwtauthority.Unmarshal(secret.Data["pool"])
6165
if err != nil {
6266
t.Fatal(err)
@@ -108,9 +112,6 @@ func poolState(t *testing.T, kc *fake.Clientset) ([]string, string) {
108112
if err != nil {
109113
t.Fatal(err)
110114
}
111-
if jwks, err := pool.JWKS(); err != nil || !bytes.Equal(jwks, secret.Data["jwks"]) {
112-
t.Errorf("jwks = %s, want the pool's public keys (%v)", secret.Data["jwks"], err)
113-
}
114115
var ids []string
115116
for _, a := range pool.Authorities {
116117
ids = append(ids, a.ID)

‎cmd/objectstore-plugin/README.md‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,17 @@ The JWT is the one ate-api-server mints for atelet through
3232
`WorkerService.MintWorkerActorJwt`, with the audience named on the
3333
`actor_jwt` proto field. Its shape is defined in `internal/actoridjwt`, and
3434
the exchange accepts only the worker-for-actor kind, never an actor's own
35-
token. Its public keys come from the `jwks` key of the `actor-id-jwt-pool`
36-
Secret, read through `oidcjwt.FileKeySource`.
35+
token. The exchange fetches the issuer's public keys through OpenID Connect
36+
discovery at `<issuer>/.well-known/openid-configuration`, at sidecar start
37+
and then as `oidcjwt.Verifier` refreshes them. An issuer it cannot reach
38+
fails the exchange with `UNAVAILABLE`.
39+
The issuer's certificate must verify against the system roots or the CA
40+
bundle `--actor-jwt-issuer-ca-file` names, by default the service DNS trust
41+
bundle. The sidecar re-reads the bundle when it changes, so a rotated bundle
42+
takes effect without a restart. If a re-read fails, it keeps the last bundle
43+
and logs a warning. If the issuer's certificate does not verify at start, the
44+
sidecar exits. Any other error at start is logged, and the sidecar serves and
45+
fetches again on the next exchange.
3746

3847
Four packages behind the exchange know no storage provider and are what a
3948
provider's own plugin binary reuses. The rest is this binary's.
@@ -42,7 +51,7 @@ Reusable by every provider:
4251

4352
- `internal/objectstoreplugin/actorauthz`: the checks every exchange applies
4453
before it issues a credential, built on the three below.
45-
- `internal/oidcjwt`: verifies the JWT against the `jwks` file.
54+
- `internal/oidcjwt`: verifies the JWT against the issuer's published keys.
4655
- `internal/objectstoreplugin/actorlookup`: fetches the actor over the
4756
loopback, refuses a recreated one.
4857
- `internal/objectstoreplugin/actorscope`: turns the actor's status into

‎cmd/objectstore-plugin/exchange.go‎

Lines changed: 43 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"fmt"
2222
"log/slog"
2323
"net"
24+
"time"
2425

2526
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
2627
"google.golang.org/grpc"
@@ -44,7 +45,7 @@ type exchangeConfig struct {
4445
listen string
4546
credBundle string
4647
issuer string
47-
jwksFile string
48+
issuerCAFile string
4849
transferAccount string
4950
ateapi actorlookup.Config
5051
}
@@ -63,19 +64,16 @@ func newExchangeService(ctx context.Context, cfg exchangeConfig) (exchangepb.Cre
6364
if cfg.issuer == "" {
6465
return nil, nil, errors.New("--actor-jwt-issuer is required to serve the exchange on GCS")
6566
}
66-
if cfg.jwksFile == "" {
67-
return nil, nil, errors.New("--actor-jwt-jwks-file is required to serve the exchange on GCS")
68-
}
6967
if cfg.transferAccount == "" {
7068
return nil, nil, errors.New("--transfer-service-account is required to serve the exchange on GCS")
7169
}
72-
// Read the key set once now, so a wrong path fails the start rather
73-
// than the first exchange.
74-
keys := oidcjwt.FileKeySource(cfg.jwksFile)
75-
if _, err := keys.Keys(ctx); err != nil {
76-
return nil, nil, fmt.Errorf("while reading the actor JWT key set: %w", err)
70+
if cfg.issuerCAFile == "" {
71+
return nil, nil, errors.New("--actor-jwt-issuer-ca-file is required to serve the exchange on GCS")
72+
}
73+
verifier, err := newActorVerifier(ctx, cfg.issuer, cfg.issuerCAFile)
74+
if err != nil {
75+
return nil, nil, err
7776
}
78-
verifier := oidcjwt.NewVerifier(cfg.issuer, []string{actorauthz.Audience}, keys, oidcjwt.WithSubstrateClaims())
7977
lookup, err := actorlookup.New(cfg.ateapi)
8078
if err != nil {
8179
return nil, nil, err
@@ -88,6 +86,41 @@ func newExchangeService(ctx context.Context, cfg exchangeConfig) (exchangepb.Cre
8886
return exchange.New(actorauthz.New(verifier, lookup), issuer), func() { _ = lookup.Close() }, nil
8987
}
9088

89+
// newActorVerifier returns the exchange's actor JWT verifier, with the
90+
// issuer's keys already fetched by checkIssuerCertificate. The verifier
91+
// trusts the system roots and caFile, re-read when it changes.
92+
func newActorVerifier(ctx context.Context, issuer, caFile string) (*oidcjwt.Verifier, error) {
93+
client, err := oidcjwt.NewHTTPClientWithRoots(credbundle.SystemPoolLoader(caFile))
94+
if err != nil {
95+
return nil, fmt.Errorf("while setting up the actor JWT issuer client: %w", err)
96+
}
97+
verifier := actorauthz.NewVerifier(issuer, client)
98+
if err := checkIssuerCertificate(ctx, issuer, caFile, verifier); err != nil {
99+
return nil, err
100+
}
101+
return verifier, nil
102+
}
103+
104+
// issuerCheckTimeout bounds the start-time fetch from the actor JWT issuer.
105+
const issuerCheckTimeout = 10 * time.Second
106+
107+
// checkIssuerCertificate fetches the issuer's keys into verifier's cache so a
108+
// certificate the client does not trust fails the start instead of every
109+
// exchange. Other errors may be transient, such as an issuer that is not up
110+
// yet, so they are logged and the exchange serves.
111+
func checkIssuerCertificate(ctx context.Context, issuer, caFile string, verifier *oidcjwt.Verifier) error {
112+
ctx, cancel := context.WithTimeout(ctx, issuerCheckTimeout)
113+
defer cancel()
114+
err := verifier.Refresh(ctx, time.Now())
115+
if certErr := (*tls.CertificateVerificationError)(nil); errors.As(err, &certErr) {
116+
return fmt.Errorf("actor JWT issuer serves a certificate that does not verify against the system roots or %s: %w", caFile, err)
117+
}
118+
if err != nil {
119+
slog.WarnContext(ctx, "Could not fetch actor JWT signing keys at start; exchanges retry the fetch", slog.String("issuer", issuer), slog.Any("err", err))
120+
}
121+
return nil
122+
}
123+
91124
// exchangeTransportCredentials serves the pod's service DNS certificate,
92125
// re-read on rotation. The exchange runs TLS without client certificates:
93126
// the JWT is the credential, and proof that the caller is atelet is a later

0 commit comments

Comments
 (0)