Skip to content

Commit 37d3df4

Browse files
committed
Give the boot-fetch golden its policy through goldenEgressPolicy
The fixture template sets goldenEgressPolicy, so the controller creates the golden actor's EgressPolicy before it resumes the actor. This drops WithGoldenEgressPolicy and the boot fetch retries that covered a policy landing after the golden boot started. The test needs the GoldenEgressPolicy preview gate. Signed-off-by: QuentinBisson <[email protected]>
1 parent 3a06d83 commit 37d3df4

5 files changed

Lines changed: 94 additions & 99 deletions

File tree

‎internal/e2e/fixture.go‎

Lines changed: 0 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,6 @@ func decodeSubstrateTemplates(t *testing.T, rendered []byte) []*ateapipb.ActorTe
9292
type FixtureOption func(*fixtureConfig)
9393

9494
type fixtureConfig struct {
95-
goldenEgress []*ateapipb.EgressRule
9695
templateValues map[string]string
9796
}
9897

@@ -107,38 +106,6 @@ func WithTemplateValue(name, value string) FixtureOption {
107106
}
108107
}
109108

110-
// WithGoldenEgressPolicy gives each template's golden actor an EgressPolicy
111-
// with rules, for a fixture whose workload reaches the network before it is
112-
// ready. The gateway denies an actor with no policy, and the golden actor
113-
// exists only once its template does, so the policy lands after the golden
114-
// boot has started: the workload must retry what it fetches.
115-
func WithGoldenEgressPolicy(rules ...*ateapipb.EgressRule) FixtureOption {
116-
return func(c *fixtureConfig) { c.goldenEgress = rules }
117-
}
118-
119-
// ensureGoldenEgressPolicy waits for tmpl's golden actor to exist and gives it
120-
// an EgressPolicy with rules. Deleting the template deletes the golden actor
121-
// and its policy with it.
122-
func ensureGoldenEgressPolicy(t *testing.T, ctx context.Context, clients *Clients, tmpl *ateapipb.ActorTemplate, rules []*ateapipb.EgressRule) {
123-
t.Helper()
124-
golden := &ateapipb.ObjectRef{Atespace: tmpl.GetMetadata().GetAtespace(), Name: tmpl.GetMetadata().GetUid()}
125-
deadline := time.Now().Add(2 * time.Minute)
126-
for {
127-
_, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{Actor: golden})
128-
if err == nil {
129-
break
130-
}
131-
if status.Code(err) != codes.NotFound {
132-
t.Fatalf("GetActor for the golden actor of %s/%s: %v", tmpl.GetMetadata().GetAtespace(), tmpl.GetMetadata().GetName(), err)
133-
}
134-
if time.Now().After(deadline) {
135-
t.Fatalf("golden actor of %s/%s was not created within 2m", tmpl.GetMetadata().GetAtespace(), tmpl.GetMetadata().GetName())
136-
}
137-
time.Sleep(500 * time.Millisecond)
138-
}
139-
EnsureEgressPolicy(t, ctx, clients, golden, rules...)
140-
}
141-
142109
// SubstrateFixtureManifests names the two manifest templates a substrate
143110
// fixture is built from (both repo-relative, under internal/e2e/fixtures).
144111
type SubstrateFixtureManifests struct {
@@ -236,12 +203,6 @@ func DeploySubstrateFixture(t *testing.T, ctx context.Context, clients *Clients,
236203
created = append(created, c)
237204
}
238205

239-
if cfg.goldenEgress != nil {
240-
for _, tmpl := range created {
241-
ensureGoldenEgressPolicy(t, ctx, clients, tmpl, cfg.goldenEgress)
242-
}
243-
}
244-
245206
for _, tmpl := range created {
246207
t.Logf("Waiting for ActorTemplate %s/%s golden snapshot...", atespace, tmpl.GetMetadata().GetName())
247208
WaitForSubstrateTemplateReady(ctx, t, clients, atespace, tmpl.GetMetadata().GetName())

‎internal/e2e/fixtures/probe/main.go‎

Lines changed: 13 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -479,17 +479,14 @@ func writeJSON(w http.ResponseWriter, v any) {
479479
// it proves the actor had egress while it booted.
480480
const bootFetchURLEnv = "PROBE_BOOT_FETCH_URL"
481481

482-
// bootFetchBudget bounds the boot fetch, inside the template's wakeupProbe
482+
// bootFetchTimeout bounds the boot fetch, inside the template's wakeupProbe
483483
// timeout.
484-
const bootFetchBudget = 45 * time.Second
485-
486-
// bootFetch GETs url, trusting only trustBundle, until it answers 200 or
487-
// budget runs out. It retries because the gateway caches a missing
488-
// EgressPolicy for its TTL, and a suite can only give the golden actor one
489-
// once the actor exists.
490-
func bootFetch(ctx context.Context, url, trustBundle string, budget time.Duration) error {
491-
ctx, cancel := context.WithTimeout(ctx, budget)
492-
defer cancel()
484+
const bootFetchTimeout = 30 * time.Second
485+
486+
// bootFetch GETs url once, trusting only trustBundle, and fails unless it
487+
// answers 200. It does not retry: the actor's EgressPolicy exists before the
488+
// actor resumes, so a failure is the finding.
489+
func bootFetch(ctx context.Context, url, trustBundle string) error {
493490
b, err := os.ReadFile(trustBundle)
494491
if err != nil {
495492
return fmt.Errorf("reading trust bundle: %w", err)
@@ -499,22 +496,14 @@ func bootFetch(ctx context.Context, url, trustBundle string, budget time.Duratio
499496
return fmt.Errorf("no certificates parsed from %s", trustBundle)
500497
}
501498
client := &http.Client{
502-
Timeout: 10 * time.Second,
499+
Timeout: bootFetchTimeout,
503500
Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool}},
504501
}
505502
defer client.CloseIdleConnections()
506-
for {
507-
lastErr := fetchOnce(ctx, client, url)
508-
if lastErr == nil {
509-
return nil
510-
}
511-
log.Printf("probe: boot fetch of %s: %v", url, lastErr)
512-
select {
513-
case <-ctx.Done():
514-
return fmt.Errorf("boot fetch of %s: %w", url, lastErr)
515-
case <-time.After(time.Second):
516-
}
503+
if err := fetchOnce(ctx, client, url); err != nil {
504+
return fmt.Errorf("boot fetch of %s: %w", url, err)
517505
}
506+
return nil
518507
}
519508

520509
// restoreFetch answers the wakeup probe once the actor named in its identity
@@ -552,7 +541,7 @@ func (f *restoreFetch) healthz(w http.ResponseWriter, _ *http.Request) {
552541
}
553542

554543
func (f *restoreFetch) fetch(id string) {
555-
err := bootFetch(context.Background(), f.url, f.trustBundle, bootFetchBudget)
544+
err := bootFetch(context.Background(), f.url, f.trustBundle)
556545
f.mu.Lock()
557546
defer f.mu.Unlock()
558547
if f.fetching == id {
@@ -590,7 +579,7 @@ func main() {
590579
if err != nil {
591580
log.Fatalf("probe: reading %s: %v", identityFile, err)
592581
}
593-
if err := bootFetch(context.Background(), url, trustFile, bootFetchBudget); err != nil {
582+
if err := bootFetch(context.Background(), url, trustFile); err != nil {
594583
log.Fatalf("probe: %v", err)
595584
}
596585
log.Printf("probe: boot fetch of %s succeeded", url)

‎internal/e2e/fixtures/probe/main_test.go‎

Lines changed: 68 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -380,37 +380,37 @@ func writeServerTrustBundle(t *testing.T, server *httptest.Server) string {
380380
return path
381381
}
382382

383-
// A golden actor's EgressPolicy can land after its boot started, and the
384-
// gateway caches the missing policy for a while, so the boot fetch retries
385-
// until the origin answers.
386-
func TestBootFetchRetriesUntilOK(t *testing.T) {
387-
var calls atomic.Int32
388-
origin := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
389-
if calls.Add(1) < 3 {
390-
w.WriteHeader(http.StatusForbidden)
391-
return
392-
}
393-
w.WriteHeader(http.StatusOK)
394-
}))
395-
defer origin.Close()
396-
397-
if err := bootFetch(t.Context(), origin.URL, writeServerTrustBundle(t, origin), 30*time.Second); err != nil {
398-
t.Fatalf("bootFetch() error = %v, want nil", err)
399-
}
400-
if got := calls.Load(); got != 3 {
401-
t.Errorf("origin saw %d requests, want 3", got)
383+
// The actor's EgressPolicy exists before it resumes, so the boot fetch makes
384+
// one request and reports its failure.
385+
func TestBootFetch(t *testing.T) {
386+
tests := []struct {
387+
name string
388+
status int
389+
wantErr string
390+
}{
391+
{name: "origin answers 200", status: http.StatusOK},
392+
{name: "origin denies", status: http.StatusForbidden, wantErr: "status 403"},
402393
}
403-
}
404-
405-
func TestBootFetchFailsAfterBudget(t *testing.T) {
406-
origin := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
407-
w.WriteHeader(http.StatusForbidden)
408-
}))
409-
defer origin.Close()
410-
411-
err := bootFetch(t.Context(), origin.URL, writeServerTrustBundle(t, origin), 1500*time.Millisecond)
412-
if err == nil || !strings.Contains(err.Error(), "status 403") {
413-
t.Fatalf("bootFetch() error = %v, want the last status", err)
394+
for _, tt := range tests {
395+
t.Run(tt.name, func(t *testing.T) {
396+
var calls atomic.Int32
397+
origin := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
398+
calls.Add(1)
399+
w.WriteHeader(tt.status)
400+
}))
401+
defer origin.Close()
402+
403+
err := bootFetch(t.Context(), origin.URL, writeServerTrustBundle(t, origin))
404+
if tt.wantErr == "" && err != nil {
405+
t.Fatalf("bootFetch() error = %v, want nil", err)
406+
}
407+
if tt.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tt.wantErr)) {
408+
t.Fatalf("bootFetch() error = %v, want %q", err, tt.wantErr)
409+
}
410+
if got := calls.Load(); got != 1 {
411+
t.Errorf("origin saw %d requests, want 1", got)
412+
}
413+
})
414414
}
415415
}
416416

@@ -448,3 +448,41 @@ func TestRestoreFetchGatesHealthzOnIdentity(t *testing.T) {
448448
t.Fatalf("healthz as a restored actor after it fetched = %d, want 200", got)
449449
}
450450
}
451+
452+
// A failed restore fetch leaves healthz at 503, and the next wakeup probe
453+
// starts another fetch.
454+
func TestRestoreFetchRetriesOnNextProbe(t *testing.T) {
455+
var deny atomic.Bool
456+
deny.Store(true)
457+
origin := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
458+
if deny.Load() {
459+
w.WriteHeader(http.StatusForbidden)
460+
return
461+
}
462+
w.WriteHeader(http.StatusOK)
463+
}))
464+
defer origin.Close()
465+
identity := filepath.Join(t.TempDir(), "actor-id")
466+
if err := os.WriteFile(identity, []byte("restored"), 0o600); err != nil {
467+
t.Fatal(err)
468+
}
469+
f := &restoreFetch{url: origin.URL, identityFile: identity, trustBundle: writeServerTrustBundle(t, origin), fetched: "golden"}
470+
healthz := func() int {
471+
rec := httptest.NewRecorder()
472+
f.healthz(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil))
473+
return rec.Code
474+
}
475+
476+
f.fetch("restored")
477+
if got := healthz(); got != http.StatusServiceUnavailable {
478+
t.Fatalf("healthz after a failed fetch = %d, want 503", got)
479+
}
480+
deny.Store(false)
481+
deadline := time.Now().Add(10 * time.Second)
482+
for healthz() != http.StatusOK {
483+
if time.Now().After(deadline) {
484+
t.Fatal("healthz did not answer 200 after the origin allowed the fetch")
485+
}
486+
time.Sleep(50 * time.Millisecond)
487+
}
488+
}

‎internal/e2e/fixtures/probe/probe-bootfetch-template.yaml.tmpl‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,9 @@
1616
# ateapipb.ActorTemplate whose probe fetches the suite's BOOT_FETCH_URL through the
1717
# egress gateway before it listens, so its golden snapshot exists only if the
1818
# actor had egress while it booted. Selects the probe pool of probe.yaml.tmpl;
19-
# the suite gives the golden actor its EgressPolicy (see
20-
# e2e.WithGoldenEgressPolicy), the template the trust bundle, and the URL (see
21-
# e2e.WithTemplateValue).
19+
# goldenEgressPolicy (preview gate GoldenEgressPolicy) allows the golden actor
20+
# to reach BOOT_FETCH_HOST; the suite gives the template the trust bundle and
21+
# BOOT_FETCH_HOST and BOOT_FETCH_URL (see e2e.WithTemplateValue).
2222

2323
metadata:
2424
atespace: ate-e2e-probe${FIXTURE_SUFFIX}
@@ -36,7 +36,7 @@ containers:
3636
volumeMounts:
3737
- name: system-info
3838
mountPath: /run/ate
39-
# Longer than the probe's own boot-fetch budget, so a failed fetch is the
39+
# Longer than the probe's boot fetch timeout, so a failed fetch is the
4040
# probe's logged error rather than a probe timeout.
4141
wakeupProbe:
4242
httpGet:
@@ -52,6 +52,10 @@ volumes:
5252
- field: ACTOR_METADATA_FIELD_NAME
5353
path: actor-id
5454
${TEMPLATE_TRUST_BUNDLE}
55+
goldenEgressPolicy:
56+
rules:
57+
- https:
58+
hostnames: ["${BOOT_FETCH_HOST}"]
5559
resources:
5660
limits:
5761
cpuMillicores: 1000

‎internal/e2e/suites/egressmitm/goldenboot_test.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"testing"
2020

2121
"go.ate.dev/substrate/internal/e2e"
22+
"go.ate.dev/substrate/internal/preview"
2223
"go.ate.dev/substrate/pkg/proto/ateapipb"
2324
)
2425

@@ -32,11 +33,13 @@ var bootFetchManifests = e2e.SubstrateFixtureManifests{
3233
// gateway before it listens, and exits if it cannot, so its template gets a
3334
// golden snapshot only if the worker tunneled the golden actor's egress
3435
// before the wakeup probe and the gateway admitted the actor while it was
35-
// still resuming. An actor restored from that snapshot then has to fetch again
36+
// still resuming. The template's goldenEgressPolicy gives the golden actor its
37+
// policy before the controller resumes it. An actor restored from that snapshot then has to fetch again
3638
// as itself before its wakeup probe passes, which covers the restore paths.
3739
//
3840
// hack/run-e2e-kind.sh ./internal/e2e/suites/egressmitm -run TestGoldenBootEgress -v -args --no-color
3941
func TestGoldenBootEgress(t *testing.T) {
42+
e2e.RequirePreview(t, preview.GateGoldenEgressPolicy)
4043
env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO")
4144
if err != nil {
4245
t.Fatalf("CheckEnv failed: %v", err)
@@ -49,7 +52,7 @@ func TestGoldenBootEgress(t *testing.T) {
4952
// DeploySubstrateFixture returns once every template has its golden
5053
// snapshot, which is the assertion.
5154
atespace, _ := e2e.DeploySubstrateFixture(t, ctx, clients, bootFetchManifests, env["BUCKET_NAME"], "egressmitm-bootfetch", []string{e2e.EgressTrustBundleName},
52-
e2e.WithGoldenEgressPolicy(e2e.EgressAllowHTTPS(origin.SvcName())),
55+
e2e.WithTemplateValue("BOOT_FETCH_HOST", origin.SvcName()),
5356
e2e.WithTemplateValue("BOOT_FETCH_URL", "https://"+origin.SvcName()+"/healthz"))
5457

5558
t.Run("restore", func(t *testing.T) {

0 commit comments

Comments
 (0)