Skip to content

Commit e73bfd6

Browse files
committed
objectstore: warn on missing buckets and cover actor deletion
1 parent 79bb4a8 commit e73bfd6

3 files changed

Lines changed: 136 additions & 1 deletion

File tree

‎cmd/ateapi/internal/controlapi/functionaltest/actor_test.go‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,16 +22,20 @@ import (
2222
"fmt"
2323
"strings"
2424
"sync"
25+
"sync/atomic"
2526
"testing"
2627
"time"
2728

29+
"cloud.google.com/go/storage"
2830
"github.com/agent-substrate/substrate/cmd/ateapi/internal/store"
2931
"github.com/agent-substrate/substrate/internal/actoridjwt"
3032
"github.com/agent-substrate/substrate/internal/ateattr"
33+
"github.com/agent-substrate/substrate/internal/objectstore"
3134
"github.com/agent-substrate/substrate/internal/proto/ateletpb"
3235
"github.com/agent-substrate/substrate/internal/resources"
3336
"github.com/agent-substrate/substrate/internal/volume"
3437
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
38+
"github.com/aws/aws-sdk-go-v2/service/s3/types"
3539
"github.com/google/go-cmp/cmp"
3640
"github.com/google/go-cmp/cmp/cmpopts"
3741
"go.opentelemetry.io/otel/attribute"
@@ -1080,6 +1084,81 @@ func TestDeleteActor_Success(t *testing.T) {
10801084
assertGrpcError(t, err, codes.NotFound, "Actor test-atespace/id1 not found")
10811085
}
10821086

1087+
func TestDeleteActor_MissingSnapshotBucket(t *testing.T) {
1088+
for _, provider := range []struct {
1089+
name string
1090+
err error
1091+
}{
1092+
{name: "S3", err: &types.NoSuchBucket{}},
1093+
{name: "GCS", err: storage.ErrBucketNotExist},
1094+
} {
1095+
for _, state := range []ateapipb.ActorState{
1096+
ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
1097+
ateapipb.ActorState_ACTOR_STATE_DELETING,
1098+
} {
1099+
t.Run(provider.name+"/"+state.String(), func(t *testing.T) {
1100+
ns := namespaceForTest("ns-delete-missing-bucket")
1101+
tc := setupTest(t, ns)
1102+
defer tc.cleanup()
1103+
createTemplate(t, tc, ns)
1104+
ctx := t.Context()
1105+
ref := &ateapipb.ObjectRef{Atespace: testAtespace, Name: testActorID}
1106+
actor, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
1107+
Metadata: &ateapipb.ResourceMetadata{Atespace: ref.GetAtespace(), Name: ref.GetName()},
1108+
ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"},
1109+
}})
1110+
if err != nil {
1111+
t.Fatalf("CreateActor: %v", err)
1112+
}
1113+
uri, err := resources.NewActorSnapshotURI(testStorageLocation, testAtespace, actor.GetMetadata().GetUid(), "snapshot")
1114+
if err != nil {
1115+
t.Fatal(err)
1116+
}
1117+
if _, err := tc.persistence.UpdateActor(ctx, resources.ActorRefFromActor(actor), store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error {
1118+
toUpdate.Status.State = state
1119+
toUpdate.Status.ExternalSnapshot = nil
1120+
if state == ateapipb.ActorState_ACTOR_STATE_SUSPENDED {
1121+
toUpdate.Status.ExternalSnapshot = &ateapipb.ExternalSnapshot{
1122+
SnapshotUri: uri.String(), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL,
1123+
}
1124+
} else {
1125+
// A failed upload leaves only the in-progress URI, even
1126+
// when a prior deletion already marked the actor DELETING.
1127+
toUpdate.Status.InProgressSnapshotUri = uri.String()
1128+
}
1129+
return nil
1130+
}); err != nil {
1131+
t.Fatalf("UpdateActor: %v", err)
1132+
}
1133+
bucket, prefix, err := objectstore.BucketPrefix(uri.OwnerPrefix())
1134+
if err != nil {
1135+
t.Fatal(err)
1136+
}
1137+
var listCalls atomic.Int32
1138+
tc.objectStore.OnList = func(gotBucket, gotPrefix string) error {
1139+
listCalls.Add(1)
1140+
if gotBucket != bucket || gotPrefix != prefix {
1141+
return fmt.Errorf("unexpected cleanup prefix: %s/%s", gotBucket, gotPrefix)
1142+
}
1143+
return fmt.Errorf("snapshot bucket removed: %w", provider.err)
1144+
}
1145+
deleted, err := tc.client.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: ref})
1146+
if err != nil {
1147+
t.Fatalf("DeleteActor with missing snapshot bucket: %v", err)
1148+
}
1149+
if deleted.GetMetadata().GetUid() != actor.GetMetadata().GetUid() {
1150+
t.Errorf("DeleteActor returned a different actor: %v", deleted)
1151+
}
1152+
if got := listCalls.Load(); got != 1 {
1153+
t.Errorf("snapshot cleanup list calls = %d, want 1", got)
1154+
}
1155+
_, err = tc.client.GetActor(ctx, &ateapipb.GetActorRequest{Actor: ref})
1156+
assertGrpcError(t, err, codes.NotFound, "Actor test-atespace/id1 not found")
1157+
})
1158+
}
1159+
}
1160+
}
1161+
10831162
func TestDeleteActor_NotSuspended(t *testing.T) {
10841163
ns := namespaceForTest("ns-delete-notsuspended")
10851164
tc := setupTest(t, ns)

‎internal/objectstore/objectstore.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"context"
2727
"errors"
2828
"fmt"
29+
"log/slog"
2930
"net/url"
3031
"strings"
3132

@@ -64,6 +65,8 @@ func DeletePrefix(ctx context.Context, s Store, uri resources.StoragePrefix) err
6465
objects, err := s.List(ctx, bucket, prefix)
6566
if err != nil {
6667
if isMissingBucket(err) {
68+
slog.WarnContext(ctx, "snapshot bucket is missing; treating prefix as already deleted",
69+
slog.String("bucket", bucket), slog.String("prefix", prefix), slog.Any("error", err))
6770
return nil
6871
}
6972
return fmt.Errorf("while listing %s: %w", uri, err)
@@ -72,7 +75,12 @@ func DeletePrefix(ctx context.Context, s Store, uri resources.StoragePrefix) err
7275
group.SetLimit(prefixConcurrency)
7376
for _, object := range objects {
7477
group.Go(func() error {
75-
if err := s.Delete(ctx, bucket, object); err != nil && !isMissingBucket(err) {
78+
if err := s.Delete(ctx, bucket, object); err != nil {
79+
if isMissingBucket(err) {
80+
slog.WarnContext(ctx, "snapshot bucket is missing; treating object as already deleted",
81+
slog.String("bucket", bucket), slog.String("object", object), slog.Any("error", err))
82+
return nil
83+
}
7684
return fmt.Errorf("while deleting %s from bucket %s: %w", object, bucket, err)
7785
}
7886
return nil

‎internal/objectstore/objectstore_test.go‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,18 @@
1515
package objectstore_test
1616

1717
import (
18+
"bytes"
19+
"encoding/json"
1820
"errors"
21+
"log/slog"
1922
"strings"
2023
"testing"
2124

25+
"cloud.google.com/go/storage"
2226
"github.com/agent-substrate/substrate/internal/objectstore"
2327
"github.com/agent-substrate/substrate/internal/objectstore/objectstoretest"
2428
"github.com/agent-substrate/substrate/internal/resources"
29+
"github.com/aws/aws-sdk-go-v2/service/s3/types"
2530
"github.com/google/go-cmp/cmp"
2631
"github.com/google/go-cmp/cmp/cmpopts"
2732
)
@@ -143,6 +148,49 @@ func TestDeletePrefix(t *testing.T) {
143148
}
144149
}
145150

151+
func TestDeletePrefix_MissingBucketWarning(t *testing.T) {
152+
for _, provider := range []struct {
153+
name string
154+
err error
155+
}{
156+
{name: "S3", err: &types.NoSuchBucket{}},
157+
{name: "GCS", err: storage.ErrBucketNotExist},
158+
} {
159+
for _, phase := range []string{"list", "delete"} {
160+
t.Run(provider.name+"/"+phase, func(t *testing.T) {
161+
var logs bytes.Buffer
162+
previous := slog.Default()
163+
slog.SetDefault(slog.New(slog.NewJSONHandler(&logs, nil)))
164+
t.Cleanup(func() { slog.SetDefault(previous) })
165+
166+
fake := objectstoretest.New()
167+
uri := mustActorSnapshotURI(t, testLocation, "team-a", "actor-1", "snap-1")
168+
fake.PutSnapshot(t, uri, "manifest.json")
169+
fail := func(string, string) error { return provider.err }
170+
if phase == "list" {
171+
fake.OnList = fail
172+
} else {
173+
fake.OnDelete = fail
174+
}
175+
if err := objectstore.DeletePrefix(t.Context(), fake, uri.Prefix()); err != nil {
176+
t.Fatalf("DeletePrefix: %v", err)
177+
}
178+
var record struct {
179+
Level string `json:"level"`
180+
Bucket string `json:"bucket"`
181+
Error string `json:"error"`
182+
}
183+
if err := json.Unmarshal(logs.Bytes(), &record); err != nil {
184+
t.Fatalf("expected a warning for the missing bucket, got %q: %v", logs.String(), err)
185+
}
186+
if record.Level != "WARN" || record.Bucket != "bucket" || record.Error != provider.err.Error() {
187+
t.Errorf("missing-bucket warning = %+v, want WARN with bucket and provider error", record)
188+
}
189+
})
190+
}
191+
}
192+
}
193+
146194
func TestDeletePrefix_Failures(t *testing.T) {
147195
failure := errors.New("boom")
148196
tests := []struct {

0 commit comments

Comments
 (0)