Skip to content

Commit 74c6ab3

Browse files
Update the actor and its worker assignment in one transaction (agent-substrate#3223)
BindActorToWorker and the actor update ran as separate writes, and so did ReleaseActorFromWorker and the actor update. A crash or lost lease between them could leave a worker assignment row the actor did not name, or an actor naming a worker that did not hold it. Recovering from that needed stale claim lookups, rebinds, and release-only paths. BindActorToWorker now applies the actor update in its transaction, and UpdateActorAndRelease replaces ReleaseActorFromWorker. An actor names a worker if and only if that worker holds its assignment. Remove the recovery code this makes unreachable. HostingNode reads the node from the actor's status. BindActorToWorker returns ErrWorkerNotFound for a missing worker, so the resume can tell it from a deleted actor. Fixes agent-substrate#1639 AI use: AI-assisted, I reviewed and tested it. - [x] Tests pass - [x] Docs updated --------- Signed-off-by: Scott Funkenhauser <[email protected]>
1 parent 0ddbf78 commit 74c6ab3

38 files changed

Lines changed: 1356 additions & 2042 deletions

‎cmd/ateapi/internal/controlapi/actor.go‎

Lines changed: 28 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,34 @@ func (s *RPCService) UpdateActor(ctx context.Context, req *ateapi.UpdateActorReq
287287
}
288288

289289
func (s *ServiceImpl) UpdateActor(ctx context.Context, actorRef resources.ActorRef, precondition store.Precondition, mutate func(*ateapi.Actor) error) (*ateapi.Actor, error) {
290-
storedActor, err := s.store.UpdateActor(ctx, actorRef, precondition, func(toUpdate *ateapi.Actor) error {
290+
storedActor, err := s.store.UpdateActor(ctx, actorRef, precondition, s.validatedActorMutation(ctx, mutate))
291+
if err != nil {
292+
if mapped := actorWriteError(actorRef, err); mapped != err {
293+
return nil, mapped
294+
}
295+
return nil, fmt.Errorf("while updating actor: %w", err)
296+
}
297+
return storedActor, nil
298+
}
299+
300+
// actorWriteError maps the store's errors for a write to the Actor actorRef to
301+
// API errors, and returns any other error unchanged.
302+
func actorWriteError(actorRef resources.ActorRef, err error) error {
303+
switch {
304+
case errors.Is(err, store.ErrVersionConflict), errors.Is(err, store.ErrUIDConflict):
305+
return apierror.Aborted("concurrent update conflict, please retry")
306+
case errors.Is(err, store.ErrNotFound):
307+
return apierror.NotFound("actor %s not found", actorRef)
308+
case errors.Is(err, store.ErrPreconditionRequired):
309+
return apierror.InvalidArgument("while updating actor %s: %v", actorRef, err)
310+
}
311+
return err
312+
}
313+
314+
// validatedActorMutation wraps mutate so the Actor it writes is validated as
315+
// an update of the stored Actor.
316+
func (s *ServiceImpl) validatedActorMutation(ctx context.Context, mutate func(*ateapi.Actor) error) func(*ateapi.Actor) error {
317+
return func(toUpdate *ateapi.Actor) error {
291318
// Apply the mutation function to the stored value.
292319
oldVal := proto.CloneOf(toUpdate)
293320
if err := mutate(toUpdate); err != nil {
@@ -340,23 +367,7 @@ func (s *ServiceImpl) UpdateActor(ctx context.Context, actorRef resources.ActorR
340367
}
341368

342369
return nil
343-
})
344-
if err != nil {
345-
if errors.Is(err, store.ErrVersionConflict) {
346-
return nil, apierror.Aborted("concurrent update conflict, please retry")
347-
}
348-
if errors.Is(err, store.ErrUIDConflict) {
349-
return nil, apierror.Aborted("concurrent update conflict, please retry")
350-
}
351-
if errors.Is(err, store.ErrNotFound) {
352-
return nil, apierror.NotFound("actor %s not found", actorRef)
353-
}
354-
if errors.Is(err, store.ErrPreconditionRequired) {
355-
return nil, apierror.InvalidArgument("while updating actor %s: %v", actorRef, err)
356-
}
357-
return nil, fmt.Errorf("while updating actor: %w", err)
358370
}
359-
return storedActor, nil
360371
}
361372

362373
// validateTemplateVolumesUnchanged rejects a template repoint that changes

‎cmd/ateapi/internal/controlapi/actor_test.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ func TestValidateActorStatusCrash(t *testing.T) {
4343
return withActorStatus(func(s *ateapi.ActorStatus) {
4444
s.State = ateapi.ActorState_ACTOR_STATE_CRASHED
4545
s.Crash = &ateapi.ActorCrash{
46-
Message: crashMessageWorkerGone,
46+
Message: crashMessageWorkerDraining,
4747
CrashTime: &timestamppb.Timestamp{Seconds: 867},
4848
}
4949
for _, m := range mutate {
@@ -767,7 +767,11 @@ func TestUpdateActor_DeleteRecreateRace(t *testing.T) {
767767
racing := &conflictInjectingStore{
768768
Interface: persistence,
769769
inject: func() {
770-
if _, err := persistence.MarkActorForDeletion(ctx, actorRef, store.DeletePreconditions{}, nil); err != nil {
770+
// A delete releases the Actor's worker before removing it.
771+
if _, err := persistence.MarkActorForDeletion(ctx, actorRef, store.DeletePreconditions{}, func(a *ateapi.Actor) error {
772+
a.Status.WorkerAssignment = nil
773+
return nil
774+
}); err != nil {
771775
t.Fatalf("racing writer: mark for deletion: %v", err)
772776
}
773777
if _, err := persistence.DeleteActor(ctx, actorRef, store.DeletePreconditions{}); err != nil {

‎cmd/ateapi/internal/controlapi/common_test.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020

2121
"go.ate.dev/substrate/api/ateapi/v1"
2222
"go.ate.dev/substrate/cmd/ateapi/internal/store"
23+
"go.ate.dev/substrate/cmd/ateapi/internal/store/storetest"
2324
"google.golang.org/protobuf/testing/protocmp"
2425
"k8s.io/apimachinery/pkg/util/validation/field"
2526
)
@@ -65,8 +66,5 @@ func seedAssignment(t *testing.T, st store.Interface, workerName string, assignm
6566
if assignment == nil {
6667
return
6768
}
68-
ctx := context.Background()
69-
if err := st.BindActorToWorker(ctx, workerName, assignment, nil); err != nil {
70-
t.Fatalf("seed assignment on worker %q: %v", workerName, err)
71-
}
69+
storetest.MustSeedAssignment(t, context.Background(), st, workerName, assignment)
7270
}

‎cmd/ateapi/internal/controlapi/crash.go‎

Lines changed: 8 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ package controlapi
1616

1717
import (
1818
"context"
19-
"errors"
2019
"fmt"
2120
"log/slog"
2221
"strings"
@@ -35,15 +34,12 @@ import (
3534
// Fixed messages recorded in ActorCrash.message for crashes due to non-atelet
3635
// reasons. A failed atelet call builds its message with ateletCrashMessage instead.
3736
const (
38-
crashMessageWorkerAssignmentMissing = "actor has no worker assignment in a state that requires one"
3937
crashMessageLocalSnapshotNodeUnknown = "node holding the actor's local snapshot is unknown"
4038
crashMessageLocalSnapshotNodeGone = "node holding the actor's latest snapshot no longer exists"
4139
crashMessageLocalSnapshotMissing = "actor has no local snapshot"
4240
crashMessageDurableSnapshotMissing = "actor has no durable snapshot"
4341
crashMessageSnapshotStateConflict = "atelet's snapshot state conflicts with the actor's record"
44-
crashMessageWorkerGone = "assigned worker no longer exists"
4542
crashMessageWorkerDraining = "assigned worker is draining"
46-
crashMessageWorkerReassigned = "assigned worker no longer hosts the actor"
4743
crashMessageWorkerIneligible = "assigned worker no longer satisfies the actor's placement constraints"
4844
crashMessageWorkerPodGone = "worker pod went away while hosting the actor"
4945
crashMessageWorkerRestarted = "worker restarted while hosting the actor"
@@ -129,29 +125,20 @@ func crashActor(ctx context.Context, st crashActorStore, actorRef resources.Acto
129125
return fmt.Errorf("while loading actor to crash: %w", err)
130126
}
131127

128+
// An actor crashed by a writer that took over after this workflow lost its
129+
// lease keeps that first crash, and is not counted again.
132130
wasAlreadyCrashed := actor.GetStatus().GetState() == ateapi.ActorState_ACTOR_STATE_CRASHED
133131
opName = ateattr.NormalizeOperationName(opName)
134132

135-
// Release the worker before moving the actor to CRASHED state.
136-
// If the release fails we must not clear the actor's worker assignment or
137-
// mark it CRASHED: doing so would strand the still-assigned worker with no
138-
// actor referencing it, so nothing would ever retry the release and the
139-
// worker slot would be consumed until its pod dies. Returning the error
140-
// instead leaves the actor (and its assignment) intact so the caller retries
141-
// crashActor, which re-attempts the release. releaseWorker is idempotent, so
142-
// a retry after a release that already succeeded is a no-op.
143-
if _, err := releaseWorker(ctx, st, actor); err != nil {
144-
return fmt.Errorf("while releasing worker to crash actor: %w", err)
145-
}
146-
147133
// Snapshot crash attributes before pod and pool pointers are cleared below;
148134
// the counter itself is emitted only after the transition commits.
149135
sandboxClass := actor.GetStatus().GetWorkerAssignment().GetSandboxRuntime().GetSandboxClass()
150136
crashAttrs := ateattr.ActorMetricAttributes(actor, sandboxClass, opName)
151137

152-
if _, err := st.UpdateActor(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapi.Actor) error {
138+
// The worker is released in the same write, so a failure leaves both the
139+
// actor and its assignment for the caller's retry.
140+
if _, err := st.UpdateActorAndRelease(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapi.Actor) error {
153141
toUpdate.Status.State = ateapi.ActorState_ACTOR_STATE_CRASHED
154-
// An actor crashed concurrently, e.g. by worker deletion, keeps its first crash.
155142
if !wasAlreadyCrashed {
156143
toUpdate.Status.Crash = newActorCrash(opName, message)
157144
}
@@ -163,15 +150,14 @@ func crashActor(ctx context.Context, st crashActorStore, actorRef resources.Acto
163150
toUpdate.Status.WorkerAssignment = nil
164151
return nil
165152
}); err != nil {
166-
return fmt.Errorf("while marking actor crashed: %w", err)
153+
return fmt.Errorf("while marking actor crashed: %w", actorWriteError(actorRef, err))
167154
}
168155

169-
// Increment metric only after a successful UpdateActor, and only if the actor was not already crashed.
156+
// Count the crash only once it has committed.
170157
if !wasAlreadyCrashed {
171158
logActorCrashed(ctx, actor, opName)
172159
recordActorCrash(ctx, crashAttrs)
173160
}
174-
175161
return nil
176162
}
177163

@@ -205,43 +191,5 @@ func logActorCrashed(ctx context.Context, actor *ateapi.Actor, opName string) {
205191
// an actor.
206192
type crashActorStore interface {
207193
GetActor(ctx context.Context, actorRef resources.ActorRef) (*ateapi.Actor, error)
208-
UpdateActor(ctx context.Context, actorRef resources.ActorRef, precondition store.Precondition, mutate func(toUpdate *ateapi.Actor) error) (*ateapi.Actor, error)
209-
GetWorker(ctx context.Context, name string) (*ateapi.Worker, error)
210-
ReleaseActorFromWorker(ctx context.Context, workerName string, actorUID string) (*ateapi.Worker, error)
211-
}
212-
213-
// releaseWorker clears the worker's assignment if it still points at the given
214-
// actor. A missing worker or an already-cleared assignment is not an error.
215-
// It returns the worker as it stands after the release, which callers holding
216-
// a cache of workers hand to it.
217-
func releaseWorker(ctx context.Context, st crashActorStore, actor *ateapi.Actor) (*ateapi.Worker, error) {
218-
assignment := actor.GetStatus().GetWorkerAssignment()
219-
if assignment == nil {
220-
slog.WarnContext(ctx, "Actor's worker assignment is already cleared")
221-
return nil, nil
222-
}
223-
workerName := assignment.GetWorker().GetName()
224-
225-
_, err := st.GetWorker(ctx, workerName)
226-
if errors.Is(err, store.ErrNotFound) {
227-
// No need to release if the worker is not found.
228-
slog.WarnContext(ctx, "Worker already gone while crashing actor, skipping release", slog.String("worker", workerName))
229-
return nil, nil
230-
}
231-
if err != nil {
232-
return nil, fmt.Errorf("while getting worker to release: %w", err)
233-
}
234-
235-
// Release only this actor's assignment; the worker may be hosting others,
236-
// and they are unaffected by this one crashing. A worker that is no longer
237-
// hosting it has already been released.
238-
released, err := st.ReleaseActorFromWorker(ctx, workerName, actor.GetMetadata().GetUid())
239-
if err != nil {
240-
return nil, fmt.Errorf("while releasing worker: %w", err)
241-
}
242-
if released == nil {
243-
slog.WarnContext(ctx, "Worker is not hosting this Actor, skipping release",
244-
slog.String("worker", workerName))
245-
}
246-
return released, nil
194+
UpdateActorAndRelease(ctx context.Context, actorRef resources.ActorRef, precondition store.Precondition, mutate func(toUpdate *ateapi.Actor) error) (*ateapi.Actor, error)
247195
}

‎cmd/ateapi/internal/controlapi/crash_test.go‎

Lines changed: 23 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -102,22 +102,6 @@ func seedWorker(t *testing.T, ctx context.Context, st store.Interface, actorRef
102102
seedAssignment(t, st, "uid", assignment)
103103
}
104104

105-
// seedUnboundActor stores a running actor whose worker-binding fields were
106-
// already cleared, e.g. by a prior release.
107-
func seedUnboundActor(t *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef) {
108-
t.Helper()
109-
storetest.MustCreateActor(t, ctx, st, &ateapi.Actor{
110-
Metadata: &ateapi.ResourceMetadata{Name: actorRef.Name, Atespace: actorRef.Atespace},
111-
Status: &ateapi.ActorStatus{
112-
State: ateapi.ActorState_ACTOR_STATE_RUNNING,
113-
LastAssignedGeneration: 1,
114-
Snapshots: []*ateapi.Snapshot{
115-
newDurableSnapshot(1, ateapi.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "reserved-snapshot", "gs://bucket/atespaces/as/actors/uid/snapshots/reserved-snapshot", ateapi.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING),
116-
},
117-
},
118-
})
119-
}
120-
121105
// assertCrashed reloads the actor and verifies it is CRASHED with its worker
122106
// binding cleared.
123107
func assertCrashed(t *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef) {
@@ -236,26 +220,6 @@ func TestCrashActor(t *testing.T) {
236220
}
237221
},
238222
},
239-
{
240-
name: "skips release for actor with no worker binding",
241-
seed: false,
242-
setup: func(t *testing.T, ctx context.Context, st store.Interface) {
243-
seedUnboundActor(t, ctx, st, actorRef)
244-
seedWorker(t, ctx, st, actorRef)
245-
},
246-
check: func(t *testing.T, ctx context.Context, st store.Interface, err error) {
247-
if err != nil {
248-
t.Fatalf("crashActor() = %v, want nil", err)
249-
}
250-
assertCrashed(t, ctx, st, actorRef)
251-
// Without a binding the worker cannot be looked up, so its
252-
// assignment must be left untouched even though it names
253-
// the crashed actor.
254-
if firstAssignment(t, st, "uid") == nil {
255-
t.Error("worker assignment = nil, want untouched")
256-
}
257-
},
258-
},
259223
{
260224
name: "actor not found",
261225
seed: false,
@@ -317,7 +281,7 @@ func TestCrashActor_RecordsCrash(t *testing.T) {
317281
}
318282

319283
// Crashing an already-crashed actor, as a concurrent crash does, keeps the first crash.
320-
if err := crashActor(ctx, st, actorRef, ateattr.OperationResume, crashMessageWorkerGone); err != nil {
284+
if err := crashActor(ctx, st, actorRef, ateattr.OperationResume, crashMessageWorkerIneligible); err != nil {
321285
t.Fatalf("second crashActor() = %v, want nil", err)
322286
}
323287
second, err := st.GetActor(ctx, actorRef)
@@ -511,8 +475,8 @@ func TestNewActorCrash(t *testing.T) {
511475
{
512476
name: "known operation prefixes the message",
513477
opName: ateattr.OperationResume,
514-
message: crashMessageWorkerGone,
515-
want: resumeOpPrefix + crashMessageWorkerGone,
478+
message: crashMessageWorkerDraining,
479+
want: resumeOpPrefix + crashMessageWorkerDraining,
516480
},
517481
{
518482
name: "unknown operation leaves the message bare",
@@ -667,61 +631,17 @@ func assertNoCrashMetricDatapoint(t *testing.T, reader *sdkmetric.ManualReader)
667631
}
668632
}
669633

670-
// failingReleaseStore wraps a store and fails every release, simulating a
671-
// transient state-store error while releasing a worker.
672-
type failingReleaseStore struct {
634+
// failingCrashStore wraps a store and fails every crash write, simulating a
635+
// transient state-store error.
636+
type failingCrashStore struct {
673637
store.Interface
674638
err error
675639
}
676640

677-
func (f failingReleaseStore) ReleaseActorFromWorker(context.Context, string, string) (*ateapi.Worker, error) {
641+
func (f failingCrashStore) UpdateActorAndRelease(context.Context, resources.ActorRef, store.Precondition, func(*ateapi.Actor) error) (*ateapi.Actor, error) {
678642
return nil, f.err
679643
}
680644

681-
// A transient failure releasing the worker must not move the actor to the
682-
// CRASHED state: doing so would strand the still-assigned worker with
683-
// no actor left to drive a retry, permanently consuming the worker slot.
684-
// crashActor must return the error with the actor and worker left intact so the
685-
// caller retries and the worker is reclaimed.
686-
func TestCrashActorReleaseFailureLeavesWorkerReclaimable(t *testing.T) {
687-
ctx := context.Background()
688-
actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"}
689-
690-
st, cleanup := storetest.SetupTestStore(t)
691-
defer cleanup()
692-
seedActor(t, ctx, st, actorRef)
693-
seedWorker(t, ctx, st, actorRef)
694-
695-
releaseErr := errors.New("state store unavailable")
696-
err := crashActor(ctx, failingReleaseStore{Interface: st, err: releaseErr}, actorRef, ateattr.OperationUnknown, "test crash")
697-
698-
if err == nil {
699-
t.Fatal("crashActor() = nil, want error")
700-
}
701-
if !errors.Is(err, releaseErr) {
702-
t.Errorf("crashActor() error = %v, want it to wrap %v", err, releaseErr)
703-
}
704-
705-
// The actor must stay RUNNING with its worker assignment intact, so a retry
706-
// can re-release the worker.
707-
got, gerr := st.GetActor(ctx, actorRef)
708-
if gerr != nil {
709-
t.Fatalf("GetActor() = %v, want nil", gerr)
710-
}
711-
if got.GetStatus().GetState() != ateapi.ActorState_ACTOR_STATE_RUNNING {
712-
t.Errorf("status = %v, want %v (actor must not be crashed when the release fails)", got.GetStatus().GetState(), ateapi.ActorState_ACTOR_STATE_RUNNING)
713-
}
714-
if got.GetStatus().GetWorkerAssignment() == nil {
715-
t.Error("WorkerAssignment cleared, want preserved so the release can be retried")
716-
}
717-
718-
// The worker must still be assigned to the actor (the failed release did not
719-
// persist): it is not leaked, and a retry will reclaim it.
720-
if firstAssignment(t, st, "uid") == nil {
721-
t.Error("worker assignment = nil, want still assigned (release failed, must remain retriable)")
722-
}
723-
}
724-
725645
// TestCrashActorAndFail verifies the error a workflow step fails with when it
726646
// crashes the actor: FailedPrecondition once the crash commits, since no retry
727647
// can finish the step, but the crash's own error when it does not, leaving the
@@ -736,7 +656,7 @@ func TestCrashActorAndFail(t *testing.T) {
736656
seedActor(t, ctx, st, actorRef)
737657
seedWorker(t, ctx, st, actorRef)
738658

739-
err := crashActorAndFail(ctx, st, actorRef, ateattr.OperationSuspend, crashMessageLocalSnapshotMissing)
659+
err := crashActorAndFail(ctx, st, actorRef, ateattr.OperationSuspend, crashMessageSnapshotStateConflict)
740660
if got := apierror.Code(err); got != codes.FailedPrecondition {
741661
t.Fatalf("apierror.Code(err) = %v, want %v (err: %v)", got, codes.FailedPrecondition, err)
742662
}
@@ -750,7 +670,7 @@ func TestCrashActorAndFail(t *testing.T) {
750670
seedWorker(t, ctx, st, actorRef)
751671

752672
releaseErr := errors.New("state store unavailable")
753-
err := crashActorAndFail(ctx, failingReleaseStore{Interface: st, err: releaseErr}, actorRef, ateattr.OperationSuspend, crashMessageLocalSnapshotMissing)
673+
err := crashActorAndFail(ctx, failingCrashStore{Interface: st, err: releaseErr}, actorRef, ateattr.OperationSuspend, crashMessageSnapshotStateConflict)
754674
if !errors.Is(err, releaseErr) {
755675
t.Fatalf("crashActorAndFail() = %v, want it to wrap %v", err, releaseErr)
756676
}
@@ -765,6 +685,20 @@ func TestCrashActorAndFail(t *testing.T) {
765685
t.Errorf("status = %v, want %v (the crash did not commit)", got.GetStatus().GetState(), ateapi.ActorState_ACTOR_STATE_RUNNING)
766686
}
767687
})
688+
689+
t.Run("conflicting crash is Aborted", func(t *testing.T) {
690+
st, cleanup := storetest.SetupTestStore(t)
691+
defer cleanup()
692+
seedActor(t, ctx, st, actorRef)
693+
seedWorker(t, ctx, st, actorRef)
694+
695+
for _, conflict := range []error{store.ErrVersionConflict, store.ErrUIDConflict} {
696+
err := crashActorAndFail(ctx, failingCrashStore{Interface: st, err: conflict}, actorRef, ateattr.OperationSuspend, crashMessageSnapshotStateConflict)
697+
if got := apierror.Code(err); got != codes.Aborted {
698+
t.Errorf("apierror.Code(err) for %v = %v, want %v (err: %v)", conflict, got, codes.Aborted, err)
699+
}
700+
}
701+
})
768702
}
769703

770704
// crashRecords captures the "Actor crashed" records a crash emits, so a test can

0 commit comments

Comments
 (0)