Skip to content

Commit 2289ade

Browse files
authored
Add pkg/objectstore/actorauthz so out-of-tree object-store providers can run the actor checks (#3358)
Step 2 of #3095. A provider in its own repository cannot import the actor checks under `internal/`. - `New(ctx, Config)` builds the verifier through `internal/objectstoreplugin/actorauthz.StartVerifier`, as the in-tree exchange does. - A `.svc` issuer is trusted only through `IssuerCAFile`, in-tree too. An install whose `.svc` issuer certificate verified only through the system roots now fails at sidecar start. - `Authorize(ctx, jwt)` returns `ActorUID`, `Grants` and `Expiry`, or a gRPC status error. A grant access with no public `Access` value gets `Internal`. - `Close` closes the ate-api-server connection and idle issuer connections. - `ateapiauth.ClientConfig` loses `K8sClient`. Its four users add the `k8sresolver` option themselves, so the package does not link client-go. - `hack/verify/package-deps.sh` bans cloud SDKs and client-go here. `hack/verify/pkg-external-import.sh` builds the package from an outside module. Tested: `hack/verify-all.sh` passes. `go test -race ./...` fails only in `internal/atunnel/egress` and `internal/netns`, as at the merge base. No e2e. - [ ] Tests pass - [x] Docs updated --------- Signed-off-by: Anish Ramasekar <[email protected]>
1 parent db46fd5 commit 2289ade

21 files changed

Lines changed: 1325 additions & 178 deletions

File tree

‎cmd/atecontroller/main.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
"go.ate.dev/substrate/internal/ateapiauth"
2929
"go.ate.dev/substrate/internal/egresspolicy"
3030
"go.ate.dev/substrate/internal/installdefaults"
31+
"go.ate.dev/substrate/internal/k8sresolver"
3132
"go.ate.dev/substrate/internal/preview"
3233
"go.ate.dev/substrate/internal/serverboot"
3334
"go.ate.dev/substrate/internal/version"
@@ -209,7 +210,6 @@ func main() {
209210
}
210211

211212
dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{
212-
K8sClient: k8sClient,
213213
CAFile: *ateapiCAFile,
214214
ServerName: *ateapiServerName,
215215
ClientCredBundle: *ateapiClientCert,
@@ -218,6 +218,7 @@ func main() {
218218
setupLog.Error(err, "building ateapi dial options")
219219
os.Exit(1)
220220
}
221+
dialOpts = append(dialOpts, grpc.WithResolvers(k8sresolver.NewBuilder(k8sClient)))
221222
dialOpts = append(dialOpts, grpc.WithStatsHandler(otelgrpc.NewClientHandler()))
222223
// atecontroller acts for the system, not a user, so it always opts in to
223224
// the preview features the server enables.

‎cmd/atelet/main.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ import (
5757
"go.ate.dev/substrate/internal/credbundle"
5858
"go.ate.dev/substrate/internal/imagecache"
5959
"go.ate.dev/substrate/internal/installdefaults"
60+
"go.ate.dev/substrate/internal/k8sresolver"
6061
"go.ate.dev/substrate/internal/nodepath"
6162
"go.ate.dev/substrate/internal/otlprelay"
6263
"go.ate.dev/substrate/internal/pluginsocket"
@@ -382,14 +383,14 @@ func main() {
382383
}
383384

384385
dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{
385-
K8sClient: k8sClient,
386386
CAFile: *ateapiCAFile,
387387
ServerName: *ateapiServerName,
388388
ClientCredBundle: *grpcServerCredBundle,
389389
})
390390
if err != nil {
391391
serverboot.Fatal(ctx, "Failed to build ateapi client credentials", err)
392392
}
393+
dialOpts = append(dialOpts, grpc.WithResolvers(k8sresolver.NewBuilder(k8sClient)))
393394
// atelet acts for the system, not a user, so it always opts in to the
394395
// preview features the server enables.
395396
dialOpts = append(dialOpts, preview.DialOptionsWithOptIn()...)

‎cmd/atenet/internal/router/router.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ import (
4343
"go.ate.dev/substrate/cmd/atenet/internal/router/ingress"
4444
"go.ate.dev/substrate/internal/ateapiauth"
4545
"go.ate.dev/substrate/internal/credbundle"
46+
"go.ate.dev/substrate/internal/k8sresolver"
4647
"go.ate.dev/substrate/internal/preview"
4748
"go.ate.dev/substrate/internal/serverboot"
4849
"go.ate.dev/substrate/internal/version"
@@ -168,14 +169,16 @@ func (s *RouterServer) Run(ctx context.Context) error {
168169
go serverboot.StartHealthServer(ctx, s.cfg.HealthAddr, readiness)
169170

170171
dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{
171-
K8sClient: s.clientset,
172172
CAFile: s.cfg.Auth.AteapiCAFile,
173173
ServerName: s.cfg.Auth.AteapiServerName,
174174
ClientCredBundle: s.cfg.Auth.AteapiClientCertPath,
175175
})
176176
if err != nil {
177177
return fmt.Errorf("building ateapi dial options: %w", err)
178178
}
179+
if s.clientset != nil {
180+
dialOpts = append(dialOpts, grpc.WithResolvers(k8sresolver.NewBuilder(s.clientset)))
181+
}
179182
dialOpts = append(dialOpts, grpc.WithStatsHandler(otelgrpc.NewClientHandler()))
180183
// atenet acts for the system, not a user, so it always opts in to the
181184
// preview features the server enables.

‎cmd/objectstore-plugin/README.md‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,10 @@ token. The exchange fetches the issuer's public keys through OpenID Connect
3636
discovery at `<issuer>/.well-known/openid-configuration`, at sidecar start
3737
and then as `oidcjwt.Verifier` refreshes them. An issuer it cannot reach
3838
fails the exchange with `UNAVAILABLE`.
39-
The issuer's certificate must verify against the system roots or the CA
40-
bundle `--actor-jwt-issuer-ca-file` names, by default the service DNS trust
41-
bundle. The sidecar re-reads the bundle when it changes, so a rotated bundle
39+
The issuer's certificate must verify against the CA bundle
40+
`--actor-jwt-issuer-ca-file` names, by default the service DNS trust bundle.
41+
For an issuer whose host ends in `.svc`, only that bundle is trusted. For any
42+
other issuer, the system roots are trusted too. The sidecar re-reads the bundle when it changes, so a rotated bundle
4243
takes effect without a restart. If a re-read fails, it keeps the last bundle
4344
and logs a warning. If the issuer's certificate does not verify at start, the
4445
sidecar exits. Any other error at start is logged, and the sidecar serves and
@@ -77,10 +78,11 @@ This binary's, GCS and S3:
7778

7879
S3 has no issuer: its node plugin ignores the JWT and transfers with the
7980
pod's own credentials, and the exchange answers `UNIMPLEMENTED` there. A
80-
provider other than these two brings its own binary in this repository, with
81-
its own exchange handler and proto shaped for its credential, its own issuer
82-
and storage client, and imports the four reusable packages. A binary outside
83-
the repository cannot, since they are under `internal/`.
81+
provider other than these two brings its own binary, with its own exchange
82+
handler and proto shaped for its credential, its own issuer and storage
83+
client. In this repository it imports the four reusable packages. Outside it,
84+
it imports `pkg/objectstore/actorauthz`, which runs the same checks and returns
85+
the grants with each prefix as a URI ending in `/`.
8486

8587
## Upload memory
8688

‎cmd/objectstore-plugin/exchange.go‎

Lines changed: 1 addition & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ import (
2121
"fmt"
2222
"log/slog"
2323
"net"
24-
"time"
2524

2625
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
2726
"google.golang.org/grpc"
@@ -34,7 +33,6 @@ import (
3433
"go.ate.dev/substrate/internal/objectstoreplugin/actorlookup"
3534
"go.ate.dev/substrate/internal/objectstoreplugin/exchange"
3635
"go.ate.dev/substrate/internal/objectstoreplugin/gcsissuer"
37-
"go.ate.dev/substrate/internal/oidcjwt"
3836
exchangepb "go.ate.dev/substrate/internal/proto/objectstoreexchangepb"
3937
"go.ate.dev/substrate/internal/serverboot"
4038
"go.ate.dev/substrate/pkg/objectstorage"
@@ -70,7 +68,7 @@ func newExchangeService(ctx context.Context, cfg exchangeConfig) (exchangepb.Cre
7068
if cfg.issuerCAFile == "" {
7169
return nil, nil, errors.New("--actor-jwt-issuer-ca-file is required to serve the exchange on GCS")
7270
}
73-
verifier, err := newActorVerifier(ctx, cfg.issuer, cfg.issuerCAFile)
71+
verifier, _, err := actorauthz.StartVerifier(ctx, cfg.issuer, cfg.issuerCAFile)
7472
if err != nil {
7573
return nil, nil, err
7674
}
@@ -86,41 +84,6 @@ func newExchangeService(ctx context.Context, cfg exchangeConfig) (exchangepb.Cre
8684
return exchange.New(actorauthz.New(verifier, lookup), issuer), func() { _ = lookup.Close() }, nil
8785
}
8886

89-
// newActorVerifier returns the exchange's actor JWT verifier, with the
90-
// issuer's keys already fetched by checkIssuerCertificate. The verifier
91-
// trusts the system roots and caFile, re-read when it changes.
92-
func newActorVerifier(ctx context.Context, issuer, caFile string) (*oidcjwt.Verifier, error) {
93-
client, err := oidcjwt.NewHTTPClientWithRoots(credbundle.SystemPoolLoader(caFile))
94-
if err != nil {
95-
return nil, fmt.Errorf("while setting up the actor JWT issuer client: %w", err)
96-
}
97-
verifier := actorauthz.NewVerifier(issuer, client)
98-
if err := checkIssuerCertificate(ctx, issuer, caFile, verifier); err != nil {
99-
return nil, err
100-
}
101-
return verifier, nil
102-
}
103-
104-
// issuerCheckTimeout bounds the start-time fetch from the actor JWT issuer.
105-
const issuerCheckTimeout = 10 * time.Second
106-
107-
// checkIssuerCertificate fetches the issuer's keys into verifier's cache so a
108-
// certificate the client does not trust fails the start instead of every
109-
// exchange. Other errors may be transient, such as an issuer that is not up
110-
// yet, so they are logged and the exchange serves.
111-
func checkIssuerCertificate(ctx context.Context, issuer, caFile string, verifier *oidcjwt.Verifier) error {
112-
ctx, cancel := context.WithTimeout(ctx, issuerCheckTimeout)
113-
defer cancel()
114-
err := verifier.Refresh(ctx, time.Now())
115-
if certErr := (*tls.CertificateVerificationError)(nil); errors.As(err, &certErr) {
116-
return fmt.Errorf("actor JWT issuer serves a certificate that does not verify against the system roots or %s: %w", caFile, err)
117-
}
118-
if err != nil {
119-
slog.WarnContext(ctx, "Could not fetch actor JWT signing keys at start; exchanges retry the fetch", slog.String("issuer", issuer), slog.Any("err", err))
120-
}
121-
return nil
122-
}
123-
12487
// exchangeTransportCredentials serves the pod's service DNS certificate,
12588
// re-read on rotation. The exchange runs TLS without client certificates:
12689
// the JWT is the credential, and proof that the caller is atelet is a later

‎cmd/objectstore-plugin/exchange_test.go‎

Lines changed: 0 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -15,21 +15,15 @@
1515
package main
1616

1717
import (
18-
"encoding/json"
1918
"encoding/pem"
2019
"net"
2120
"net/http"
2221
"net/http/httptest"
2322
"os"
2423
"path/filepath"
2524
"strings"
26-
"sync/atomic"
2725
"testing"
28-
"time"
2926

30-
"go.ate.dev/substrate/internal/actoridjwt"
31-
"go.ate.dev/substrate/internal/localjwtauthority"
32-
"go.ate.dev/substrate/internal/objectstoreplugin/actorauthz"
3327
exchangepb "go.ate.dev/substrate/internal/proto/objectstoreexchangepb"
3428
"go.ate.dev/substrate/internal/testcert"
3529
)
@@ -89,104 +83,6 @@ func TestNewExchangeService(t *testing.T) {
8983
}
9084
}
9185

92-
// TestCheckIssuerCertificate checks that errors other than an untrusted
93-
// certificate let the exchange start.
94-
func TestCheckIssuerCertificate(t *testing.T) {
95-
trusted := httptest.NewTLSServer(http.NotFoundHandler())
96-
t.Cleanup(trusted.Close)
97-
closed := httptest.NewTLSServer(http.NotFoundHandler())
98-
closed.Close()
99-
for _, tc := range []struct {
100-
name string
101-
issuer string
102-
client *http.Client
103-
}{
104-
{"unreachable issuer", closed.URL, closed.Client()},
105-
{"issuer without discovery", trusted.URL, trusted.Client()},
106-
} {
107-
t.Run(tc.name, func(t *testing.T) {
108-
if err := checkIssuerCertificate(t.Context(), tc.issuer, "ca.pem", actorauthz.NewVerifier(tc.issuer, tc.client)); err != nil {
109-
t.Fatalf("checkIssuerCertificate() = %v, want nil", err)
110-
}
111-
})
112-
}
113-
}
114-
115-
// TestNewActorVerifierWarmsCache checks that the start check fills the cache
116-
// of the verifier the exchange uses, and that a failed start check leaves the
117-
// first exchange to fetch right away.
118-
func TestNewActorVerifierWarmsCache(t *testing.T) {
119-
for _, tc := range []struct {
120-
name string
121-
// startFails makes the issuer answer 503 to the start check only.
122-
startFails bool
123-
// wantFetches is the JWKS fetch count after the start and one Verify.
124-
wantFetches int32
125-
}{
126-
{"start check fills the cache", false, 1},
127-
{"failed start check retries on first use", true, 2},
128-
} {
129-
t.Run(tc.name, func(t *testing.T) {
130-
auth, err := localjwtauthority.GenerateAuthority("ES256", "k1")
131-
if err != nil {
132-
t.Fatal(err)
133-
}
134-
pool := &localjwtauthority.ConcretePool{Authorities: []*localjwtauthority.Authority{auth}, ActiveForSigning: "k1"}
135-
keys, err := pool.VerificationKeys()
136-
if err != nil {
137-
t.Fatal(err)
138-
}
139-
jwks, err := localjwtauthority.JWKS(keys)
140-
if err != nil {
141-
t.Fatal(err)
142-
}
143-
var fetches atomic.Int32
144-
failNext := &atomic.Bool{}
145-
failNext.Store(tc.startFails)
146-
mux := http.NewServeMux()
147-
var srv *httptest.Server
148-
mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, _ *http.Request) {
149-
_ = json.NewEncoder(w).Encode(map[string]string{"issuer": srv.URL, "jwks_uri": srv.URL + "/jwks"})
150-
})
151-
mux.HandleFunc("/jwks", func(w http.ResponseWriter, _ *http.Request) {
152-
fetches.Add(1)
153-
if failNext.Swap(false) {
154-
http.Error(w, "down", http.StatusServiceUnavailable)
155-
return
156-
}
157-
_, _ = w.Write(jwks)
158-
})
159-
srv = httptest.NewTLSServer(mux)
160-
t.Cleanup(srv.Close)
161-
162-
verifier, err := newActorVerifier(t.Context(), srv.URL, writeServerCA(t, srv))
163-
if err != nil {
164-
t.Fatal(err)
165-
}
166-
now := time.Now()
167-
token, err := pool.SignJWT(&actoridjwt.Claims{
168-
Kind: actoridjwt.KindWorkerForActor,
169-
Issuer: srv.URL,
170-
Subject: actoridjwt.KindWorkerForActor.SubjectPrefix() + "/team-a/bot",
171-
Audiences: []string{actorauthz.Audience},
172-
Expiration: now.Add(time.Hour),
173-
NotBefore: now.Add(-time.Minute),
174-
IssuedAt: now,
175-
Substrate: actoridjwt.SubstrateClaims{Atespace: "team-a", ActorName: "bot", ActorUID: "uid", Node: "node-1"},
176-
})
177-
if err != nil {
178-
t.Fatal(err)
179-
}
180-
if _, err := verifier.Verify(t.Context(), token, now); err != nil {
181-
t.Fatalf("Verify() error = %v", err)
182-
}
183-
if got := fetches.Load(); got != tc.wantFetches {
184-
t.Fatalf("JWKS fetches = %d, want %d", got, tc.wantFetches)
185-
}
186-
})
187-
}
188-
}
189-
19086
// writeServerCA writes srv's certificate to a PEM file and returns its path.
19187
func writeServerCA(t *testing.T, srv *httptest.Server) string {
19288
t.Helper()

‎cmd/objectstore-plugin/main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ func main() {
8080
flags.StringVar(&xcfg.listen, "exchange-listen", "", "control mode: TCP address to serve the credential exchange on over TLS, for example :8445. Empty leaves it unserved.")
8181
flags.StringVar(&xcfg.credBundle, "exchange-cred-bundle", "/run/servicedns.podcert.ate.dev/credential-bundle.pem", "control mode: PEM credential bundle the exchange serves with, the pod's service DNS certificate")
8282
flags.StringVar(&xcfg.issuer, "actor-jwt-issuer", "", "control mode: issuer URL of the actor JWTs the exchange accepts, the value ate-api-server runs with")
83-
flags.StringVar(&xcfg.issuerCAFile, "actor-jwt-issuer-ca-file", serviceDNSTrustBundle, "control mode: CA bundle trusted in addition to the system roots when verifying --actor-jwt-issuer")
83+
flags.StringVar(&xcfg.issuerCAFile, "actor-jwt-issuer-ca-file", serviceDNSTrustBundle, "control mode: CA bundle that verifies --actor-jwt-issuer, trusted alone for a .svc host and with the system roots otherwise")
8484
flags.StringVar(&xcfg.transferAccount, "transfer-service-account", "", "control mode on GCS: email of the Google service account snapshot transfers run as. The exchange impersonates it and downscopes its token per actor")
8585
flags.StringVar(&xcfg.ateapi.Address, "ateapi-address", "127.0.0.1:443", "control mode: ate-api-server gRPC address, the loopback in the shared pod")
8686
flags.StringVar(&xcfg.ateapi.CAFile, "ateapi-ca-file", serviceDNSTrustBundle, "control mode: CA bundle used to verify ate-api-server")

‎docs/guides/operator-install.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -360,8 +360,9 @@ The exchange verifies the actor JWT with keys it fetches from `actorJWT.issuer`
360360
through OpenID Connect discovery. The default issuer is the install's
361361
ate-idp-server. A custom issuer must route to that ate-idp-server, which
362362
serves the keys the install signs actor JWTs with. It must also be reachable
363-
from the ate-api-server pod and serve a certificate that the system roots or
364-
the install's service DNS CA trust.
363+
from the ate-api-server pod and serve a certificate that the install's
364+
service DNS CA trusts. For an issuer whose host does not end in `.svc`, a
365+
certificate the system roots trust works too.
365366
The exchange fetches the keys once at start, and exits if the issuer's
366367
certificate does not verify. Then ate-api-server does not start either.
367368

‎hack/verify/package-deps.sh‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,9 @@ ateapi_banned=(
3030
github.com/aws/aws-sdk-go-v2/service/s3
3131
)
3232

33-
# Every provider's exchange imports actorauthz, so it may not depend on one
34-
# provider's wire format, credential or cloud SDK.
33+
# Every provider's exchange imports actorauthz, in-tree or through
34+
# pkg/objectstore/actorauthz, so neither may depend on one provider's wire
35+
# format, credential or cloud SDK.
3536
actorauthz_banned=(
3637
go.ate.dev/substrate/internal/proto/objectstoreexchangepb
3738
go.ate.dev/substrate/internal/objectstoreplugin/credential
@@ -45,6 +46,12 @@ actorauthz_banned=(
4546
github.com/AzureAD
4647
)
4748

49+
# Providers outside this repository import pkg/objectstore/actorauthz, so it
50+
# must not pull client-go into their builds.
51+
pkg_actorauthz_banned=(
52+
k8s.io/client-go
53+
)
54+
4855
# ko builds every binary for each platform in .ko.yaml.
4956
platforms=()
5057
while IFS= read -r platform; do
@@ -76,5 +83,6 @@ check() {
7683

7784
check ./cmd/ateapi "${ateapi_banned[@]}"
7885
check ./internal/objectstoreplugin/actorauthz "${actorauthz_banned[@]}"
86+
check ./pkg/objectstore/actorauthz "${actorauthz_banned[@]}" "${pkg_actorauthz_banned[@]}"
7987

8088
exit "${status}"

0 commit comments

Comments
 (0)