Skip to content

Commit b1d759c

Browse files
teemowQuentinBisson
authored andcommitted
Fail an actor whose image the registry refuses instead of resuming it forever
An ActorTemplate whose container image does not exist — a digest the registry answers 404 for, a private repository the pull is not allowed to read — never boots and is never reported: atelet's pull fails with the registry's answer, the Run/Restore RPC returns it untagged, ate-api treats the resume error as transient, and the ActorTemplate reconciler requeues the golden boot with backoff. The template's golden snapshot status stays empty forever, so a consumer watching it (kagent reports `Ready=False ActorTemplatePending`) cannot tell a slow boot from a misconfigured image, and every attempt pins a worker to the golden actor. Classify the registry's final word where the typed error is available. The image cache tags a *transport.Error whose HTTP status is a client error other than a timeout (408) or a throttle (429) with ateerrors.ReasonFailedGetExternalObject — the Reason the object storage clients already put on a missing snapshot object. The status decides, not the body's error code: a registry may answer a missing manifest with UNKNOWN, or a definitive 403 with a code the transport deems temporary. A 5xx, a 429 the transport's retry did not outlast, connection trouble, local cache I/O and a canceled pull pass through untagged and stay retriable. The Run and Restore boundaries claim that Reason next to ReasonInvalidContainerConfig, so ate-api crashes the actor and releases its worker. maybeCrashActor now returns the crash with its cause and the directive still attached, and the ActorTemplate reconciler, when the resume itself reports the crash, fails the template at once with that message (`GoldenActorCrashed: actor ate-golden/<uid> crashed: ... MANIFEST_UNKNOWN: manifest unknown`) instead of observing CRASHED on the retry without it. Tests: the classification table and a real-registry pull (a missing tag and a missing digest come back tagged with the registry's answer kept in the chain; a registry that only answers 503 does not), the tag through prepareOCIDirectory and the boundary's claim, maybeCrashActor's returned error, and the reconciler failing a template on a resume that crashed the golden actor. Related: agent-substrate#1220 proposes to crash actors on every failure not marked retriable and classifies registry answers on the way; this change is the narrow, status-based classification of the one class that is a registry's final word, and applies on its own.
1 parent 63fa3c3 commit b1d759c

8 files changed

Lines changed: 222 additions & 6 deletions

File tree

‎cmd/ateapi/internal/controlapi/crash.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,11 @@ func maybeCrashActor(ctx context.Context, st crashActorStore, actorRef resources
5656
slog.ErrorContext(ctx, "Failed to crash actor", slog.Any("err", cerr))
5757
return cerr
5858
}
59-
return status.Errorf(codes.DataLoss, "actor %s crashed", actorRef)
59+
// The caller learns that the actor crashed and why, with the directive
60+
// still attached: the ActorTemplate reconciler records the cause on a
61+
// golden actor's template without another round trip.
62+
return ateerrors.NewGRPCError(ctx, codes.DataLoss, ateerrors.Reason(reason), ateerrors.ActorCrashedMetadata(),
63+
fmt.Errorf("actor %s crashed: %s", actorRef, status.Convert(err).Message()))
6064
}
6165
return fmt.Errorf("%s: %w", wrapMsg, err)
6266
}

‎cmd/ateapi/internal/controlapi/crash_test.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -319,6 +319,16 @@ func TestMaybeCrashActor(t *testing.T) {
319319
if got := status.Code(err); got != codes.DataLoss {
320320
t.Errorf("status code = %v, want %v", got, codes.DataLoss)
321321
}
322+
// The caller learns that the actor crashed, why, and from what.
323+
if !ateerrors.ActorCrashRequested(err) {
324+
t.Errorf("maybeCrashActor() error = %v, want the crash directive kept", err)
325+
}
326+
if got := ateerrors.ExtractReason(err); got != string(ateerrors.ReasonTerminalFileSystemError) {
327+
t.Errorf("reason = %q, want %q", got, ateerrors.ReasonTerminalFileSystemError)
328+
}
329+
if msg := status.Convert(err).Message(); !strings.Contains(msg, "crashed") || !strings.Contains(msg, "boom") {
330+
t.Errorf("message = %q, want the crash and its cause", msg)
331+
}
322332
assertCrashed(t, ctx, st, actorRef)
323333
},
324334
},

‎cmd/ateapi/internal/controlapi/template_reconciler.go‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"time"
2323

2424
"github.com/agent-substrate/substrate/cmd/ateapi/internal/store"
25+
"github.com/agent-substrate/substrate/internal/ateerrors"
2526
"github.com/agent-substrate/substrate/internal/resources"
2627
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
2728
"google.golang.org/grpc/codes"
@@ -286,7 +287,15 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource
286287
return 0, r.tagGoldenActor(ctx, tmpl, goldenActorRef)
287288
}
288289
if _, err := r.control.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: goldenActorRef}); err != nil {
289-
// A crash during resume is observed as CRASHED on the retry.
290+
if ateerrors.ActorCrashRequested(err) {
291+
// The resume crashed the golden actor and says why — an
292+
// image the registry refuses, a container with no runnable
293+
// process. Record the cause now; the retry would observe
294+
// CRASHED without it.
295+
return 0, r.fail(ctx, tmpl, reasonGoldenActorCrashed, status.Convert(err).Message())
296+
}
297+
// Anything else is retried; a crash the resume did not report
298+
// is observed as CRASHED on the retry.
290299
return 0, fmt.Errorf("while resuming golden actor: %w", err)
291300
}
292301
deadline := time.Now().Add(goldenSnapshotWarmupFor(tmpl.GetContainers()))

‎cmd/ateapi/internal/controlapi/template_reconciler_test.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"time"
2626

2727
"github.com/agent-substrate/substrate/cmd/ateapi/internal/store"
28+
"github.com/agent-substrate/substrate/internal/ateerrors"
2829
"github.com/agent-substrate/substrate/internal/resources"
2930
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
3031
"google.golang.org/grpc/codes"
@@ -501,6 +502,19 @@ func TestReconcileOne(t *testing.T) {
501502
wantErr: true,
502503
wantResumes: 1,
503504
},
505+
{
506+
// The resume crashed the golden actor and said why (an image the
507+
// registry refuses): the template fails now, with that cause, not
508+
// on the retry that would observe CRASHED without it.
509+
name: "resume that crashes the golden actor fails the template with the cause",
510+
template: testTemplate(),
511+
control: &fakeGoldenControl{exists: true, goldenState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
512+
resumeErr: ateerrors.NewGRPCError(context.Background(), codes.DataLoss, ateerrors.ReasonFailedGetExternalObject, ateerrors.ActorCrashedMetadata(),
513+
errors.New("actor ate-golden/"+someActorUID+" crashed: while creating workload from spec: MANIFEST_UNKNOWN: manifest unknown"))},
514+
wantFailedReason: reasonGoldenActorCrashed,
515+
wantMessage: "MANIFEST_UNKNOWN: manifest unknown",
516+
wantResumes: 1,
517+
},
504518
{
505519
name: "get failure requeues",
506520
template: testTemplate(),

‎cmd/atelet/imagevolume_test.go‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ package main
1717
import (
1818
"archive/tar"
1919
"bytes"
20+
"errors"
2021
"io"
2122
"log"
2223
"net/http/httptest"
@@ -26,6 +27,8 @@ import (
2627
"strings"
2728
"testing"
2829

30+
"github.com/agent-substrate/substrate/internal/ateerrors"
31+
"github.com/agent-substrate/substrate/internal/ateompath"
2932
"github.com/agent-substrate/substrate/internal/imagecache"
3033
"github.com/agent-substrate/substrate/internal/proto/ateletpb"
3134
"github.com/google/go-containerregistry/pkg/name"
@@ -173,3 +176,32 @@ func TestResolveImageVolumes_UnmountedVolumeNotPulled(t *testing.T) {
173176
t.Errorf("resolveImageVolumes = %+v, want empty", got)
174177
}
175178
}
179+
180+
// TestPrepareOCIDirectory_RegistryRejectionIsTerminal follows an image the
181+
// registry refuses for good from the pull to the Run/Restore boundary: the
182+
// image cache's tag survives prepareOCIDirectory's wraps, and the claim the
183+
// boundaries make (CrashIfReason with ReasonFailedGetExternalObject) turns it
184+
// into a crash directive carrying that reason — so the actor is crashed with
185+
// the registry's answer instead of being resumed against the same reference
186+
// forever.
187+
func TestPrepareOCIDirectory_RegistryRejectionIsTerminal(t *testing.T) {
188+
ateompath.ActorsDir = t.TempDir()
189+
host := imageVolumeTestRegistry(t)
190+
ref := host + "/agent:does-not-exist"
191+
192+
err := prepareOCIDirectory(t.Context(), newImageVolumeStore(t), "actor-uid", "app", ref, nil, nil, nil, "", nil, nil, nil, nil)
193+
if !errors.Is(err, ateerrors.ReasonFailedGetExternalObject) {
194+
t.Fatalf("prepareOCIDirectory(%q) = %v, want it tagged ReasonFailedGetExternalObject", ref, err)
195+
}
196+
197+
claimed := ateerrors.CrashIfReason(t.Context(), err, ateerrors.ReasonInvalidContainerConfig, ateerrors.ReasonFailedGetExternalObject)
198+
if !ateerrors.ActorCrashRequested(claimed) {
199+
t.Errorf("the boundary's claim did not request a crash: %v", claimed)
200+
}
201+
if got := ateerrors.ExtractReason(claimed); got != string(ateerrors.ReasonFailedGetExternalObject) {
202+
t.Errorf("crash reason = %q, want %q", got, ateerrors.ReasonFailedGetExternalObject)
203+
}
204+
if !strings.Contains(claimed.Error(), ref) {
205+
t.Errorf("crash directive lost the image reference: %v", claimed)
206+
}
207+
}

‎cmd/atelet/main.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -519,7 +519,11 @@ func (s *AteomHerder) Run(ctx context.Context, req *ateletpb.RunRequest) (resp *
519519
if err := s.prepareOCIBundles(ctx, actorUID, actorRef,
520520
req.GetSpec(), sandboxRec.PauseImage, req.GetTargetAteomUid(),
521521
); err != nil {
522-
return nil, ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonInvalidContainerConfig)
522+
// An image the registry refuses for good (ReasonFailedGetExternalObject,
523+
// tagged by the image cache) is as terminal as a container with no
524+
// runnable process: the same reference fails the same way on every
525+
// retry, and each retry pins a worker.
526+
return nil, ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonInvalidContainerConfig, ateerrors.ReasonFailedGetExternalObject)
523527
}
524528

525529
client, err := s.dialAteom(ctx, req.GetTargetAteomUid())
@@ -1201,7 +1205,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest)
12011205
dBundles = time.Since(t)
12021206
if err != nil {
12031207
prepFailedPhase = ateattr.SnapshotPhaseOCIUnpack
1204-
return ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonTerminalFileSystemError, ateerrors.ReasonInvalidContainerConfig)
1208+
return ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonTerminalFileSystemError, ateerrors.ReasonInvalidContainerConfig, ateerrors.ReasonFailedGetExternalObject)
12051209
}
12061210
return nil
12071211
})

‎internal/imagecache/imagecache.go‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ import (
5353
"fmt"
5454
"log/slog"
5555
"net"
56+
"net/http"
5657
"os"
5758
"path/filepath"
5859
"runtime"
@@ -65,11 +66,13 @@ import (
6566
"github.com/google/go-containerregistry/pkg/name"
6667
v1 "github.com/google/go-containerregistry/pkg/v1"
6768
"github.com/google/go-containerregistry/pkg/v1/remote"
69+
"github.com/google/go-containerregistry/pkg/v1/remote/transport"
6870
"go.opentelemetry.io/otel/metric"
6971
"golang.org/x/sync/errgroup"
7072
"golang.org/x/sync/singleflight"
7173

7274
"github.com/agent-substrate/substrate/internal/ateattr"
75+
"github.com/agent-substrate/substrate/internal/ateerrors"
7376
)
7477

7578
const (
@@ -437,7 +440,7 @@ func (s *Store) EnsureImage(ctx context.Context, ref string) (_ *Image, err erro
437440
// digest, which is the only safe cache key for mutable tags.
438441
desc, headErr := remote.Head(parsedRef, s.remoteOpts(ctx, parsedRef)...)
439442
if headErr != nil {
440-
err = fmt.Errorf("while resolving tag %q to a digest: %w", ref, headErr)
443+
err = classifyRegistryErr(fmt.Errorf("while resolving tag %q to a digest: %w", ref, headErr))
441444
return nil, err
442445
}
443446
digest = desc.Digest
@@ -471,14 +474,47 @@ func (s *Store) EnsureImage(ctx context.Context, ref string) (_ *Image, err erro
471474
select {
472475
case res := <-ch:
473476
if res.Err != nil {
474-
return nil, res.Err
477+
return nil, classifyRegistryErr(res.Err)
475478
}
476479
return res.Val.(*Image), nil
477480
case <-ctx.Done():
478481
return nil, fmt.Errorf("while waiting for pull of %s: %w", digest, context.Cause(ctx))
479482
}
480483
}
481484

485+
// classifyRegistryErr tags a registry answer no retry can change — the
486+
// repository or manifest does not exist (404), the credentials do not allow
487+
// the pull (401, 403) — with ateerrors.ReasonFailedGetExternalObject, the
488+
// Reason the object storage clients put on a missing snapshot object. The RPC
489+
// boundaries in atelet claim that Reason as terminal (CrashIfReason), so an
490+
// actor whose image the registry refuses is crashed, carrying the registry's
491+
// answer, instead of being resumed against the same reference forever.
492+
// Everything else passes through untagged and stays retriable: a 5xx or a 429
493+
// the transport's retry did not outlast, connection trouble, local cache I/O,
494+
// a canceled context, and an error another layer has already tagged.
495+
func classifyRegistryErr(err error) error {
496+
if err == nil {
497+
return nil
498+
}
499+
if _, tagged := errors.AsType[ateerrors.Reason](err); tagged {
500+
return err
501+
}
502+
if regErr, ok := errors.AsType[*transport.Error](err); ok && definitiveRegistryStatus(regErr.StatusCode) {
503+
return fmt.Errorf("%w: %w", ateerrors.ReasonFailedGetExternalObject, err)
504+
}
505+
return err
506+
}
507+
508+
// definitiveRegistryStatus reports whether an HTTP status is the registry's
509+
// final word on the request: a client error other than a timeout (408) or a
510+
// throttle (429), both of which heal by themselves. The status decides, not
511+
// the body's error code: a registry may answer a missing manifest with the
512+
// code UNKNOWN, or a definitive 403 with a code the transport deems temporary.
513+
func definitiveRegistryStatus(code int) bool {
514+
return code >= http.StatusBadRequest && code < http.StatusInternalServerError &&
515+
code != http.StatusRequestTimeout && code != http.StatusTooManyRequests
516+
}
517+
482518
// cachedImageHit is the hit side of the hitMu contract: it verifies the
483519
// cached image and records last-use for eviction's LRU ordering, atomic
484520
// with respect to eviction's record removal (removeStaleRecord holds

‎internal/imagecache/imagecache_test.go‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,24 +18,30 @@ import (
1818
"archive/tar"
1919
"bytes"
2020
"context"
21+
"errors"
22+
"fmt"
2123
"io"
2224
"log"
25+
"net"
2326
"net/http"
2427
"net/http/httptest"
2528
"net/url"
2629
"os"
2730
"path/filepath"
2831
"slices"
32+
"strings"
2933
"sync/atomic"
3034
"testing"
3135
"time"
3236

37+
"github.com/agent-substrate/substrate/internal/ateerrors"
3338
"github.com/google/go-containerregistry/pkg/name"
3439
"github.com/google/go-containerregistry/pkg/registry"
3540
v1 "github.com/google/go-containerregistry/pkg/v1"
3641
"github.com/google/go-containerregistry/pkg/v1/empty"
3742
"github.com/google/go-containerregistry/pkg/v1/mutate"
3843
"github.com/google/go-containerregistry/pkg/v1/remote"
44+
"github.com/google/go-containerregistry/pkg/v1/remote/transport"
3945
"github.com/google/go-containerregistry/pkg/v1/tarball"
4046
)
4147

@@ -161,6 +167,107 @@ func TestEnsureImage_RetriesRateLimit(t *testing.T) {
161167
}
162168
}
163169

170+
// TestClassifyRegistryErr pins the split the RPC boundaries classify on: a
171+
// registry's final word (a 4xx other than a timeout or a throttle) carries
172+
// ReasonFailedGetExternalObject; everything else — a registry that keeps
173+
// failing transiently, connection trouble, a canceled pull, an error another
174+
// layer already tagged — passes through as it came.
175+
func TestClassifyRegistryErr(t *testing.T) {
176+
regErr := func(code int) error { return &transport.Error{StatusCode: code} }
177+
tests := []struct {
178+
name string
179+
err error
180+
// wantReason is the Reason the returned error must carry; empty means
181+
// it must carry none.
182+
wantReason ateerrors.Reason
183+
}{
184+
{name: "nil", err: nil},
185+
{name: "manifest unknown", err: regErr(http.StatusNotFound), wantReason: ateerrors.ReasonFailedGetExternalObject},
186+
{name: "unauthorized", err: regErr(http.StatusUnauthorized), wantReason: ateerrors.ReasonFailedGetExternalObject},
187+
{name: "denied", err: regErr(http.StatusForbidden), wantReason: ateerrors.ReasonFailedGetExternalObject},
188+
{name: "wrapped rejection", err: fmt.Errorf("in remote.Image: %w", regErr(http.StatusNotFound)), wantReason: ateerrors.ReasonFailedGetExternalObject},
189+
{name: "request timeout", err: regErr(http.StatusRequestTimeout)},
190+
{name: "throttled", err: regErr(http.StatusTooManyRequests)},
191+
{name: "server error", err: regErr(http.StatusInternalServerError)},
192+
{name: "unavailable", err: regErr(http.StatusServiceUnavailable)},
193+
{name: "connection trouble", err: &net.OpError{Op: "dial", Err: errors.New("connection refused")}},
194+
{name: "canceled", err: fmt.Errorf("while waiting for pull: %w", context.Canceled)},
195+
{name: "local cache", err: fmt.Errorf("while reading image record: %w", os.ErrPermission)},
196+
{name: "already tagged", err: fmt.Errorf("%w: %w", ateerrors.ReasonInvalidObjectURL, regErr(http.StatusNotFound)), wantReason: ateerrors.ReasonInvalidObjectURL},
197+
}
198+
for _, tt := range tests {
199+
t.Run(tt.name, func(t *testing.T) {
200+
got := classifyRegistryErr(tt.err)
201+
if tt.err == nil {
202+
if got != nil {
203+
t.Fatalf("classifyRegistryErr(nil) = %v, want nil", got)
204+
}
205+
return
206+
}
207+
if !errors.Is(got, tt.err) {
208+
t.Errorf("classifyRegistryErr(%v) = %v, want the cause kept in the chain", tt.err, got)
209+
}
210+
r, tagged := errors.AsType[ateerrors.Reason](got)
211+
switch {
212+
case tt.wantReason == "" && tagged:
213+
t.Errorf("classifyRegistryErr(%v) carries reason %v, want none (the failure must stay retriable)", tt.err, r)
214+
case tt.wantReason != "" && (!tagged || r != tt.wantReason):
215+
t.Errorf("classifyRegistryErr(%v) reason = %v (tagged=%v), want %v", tt.err, r, tagged, tt.wantReason)
216+
}
217+
})
218+
}
219+
}
220+
221+
// TestEnsureImage_RegistryRejectionIsTagged drives the classification through
222+
// a real registry: an image the registry has never seen — by tag or by digest,
223+
// a 404 the transport does not retry — comes back tagged
224+
// ReasonFailedGetExternalObject with the registry's answer still in the chain,
225+
// while a registry that keeps failing with a 5xx comes back untagged once the
226+
// retries are spent.
227+
func TestEnsureImage_RegistryRejectionIsTagged(t *testing.T) {
228+
origBackoff := dedicatedRegistryBackoff
229+
dedicatedRegistryBackoff = remote.Backoff{Duration: time.Millisecond, Factor: 2.0, Jitter: 0.1, Steps: 2}
230+
t.Cleanup(func() { dedicatedRegistryBackoff = origBackoff })
231+
232+
_, host := newTestRegistry(t)
233+
for name, ref := range map[string]string{
234+
"missing tag": host + "/test/missing:1",
235+
"missing digest": host + "/test/missing@sha256:" + strings.Repeat("0", 64),
236+
} {
237+
t.Run(name, func(t *testing.T) {
238+
_, err := newTestStore(t).EnsureImage(context.Background(), ref)
239+
if !errors.Is(err, ateerrors.ReasonFailedGetExternalObject) {
240+
t.Fatalf("EnsureImage(%q) = %v, want it tagged ReasonFailedGetExternalObject", ref, err)
241+
}
242+
regErr, ok := errors.AsType[*transport.Error](err)
243+
if !ok {
244+
t.Fatalf("EnsureImage(%q) = %v, want the registry's answer kept in the chain", ref, err)
245+
}
246+
if regErr.StatusCode != http.StatusNotFound {
247+
t.Errorf("registry answered %d, want 404", regErr.StatusCode)
248+
}
249+
})
250+
}
251+
252+
t.Run("registry that keeps failing", func(t *testing.T) {
253+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
254+
w.WriteHeader(http.StatusServiceUnavailable)
255+
}))
256+
t.Cleanup(srv.Close)
257+
u, err := url.Parse(srv.URL)
258+
if err != nil {
259+
t.Fatalf("parsing registry URL: %v", err)
260+
}
261+
_, err = newTestStore(t).EnsureImage(context.Background(), u.Host+"/test/flaky:1")
262+
if err == nil {
263+
t.Fatal("EnsureImage through a registry that only answers 503 succeeded")
264+
}
265+
if r, tagged := errors.AsType[ateerrors.Reason](err); tagged {
266+
t.Errorf("EnsureImage = %v: carries reason %v, want none (a 5xx must stay retriable)", err, r)
267+
}
268+
})
269+
}
270+
164271
func TestEnsureImage_TagPullAndDigestHit(t *testing.T) {
165272
srv, host := newTestRegistry(t)
166273
ref := host + "/test/app:latest"

0 commit comments

Comments
 (0)