Skip to content

Commit 9aaa731

Browse files
ateapi,atelet: reclaim actor node state when a worker's pod is gone
A deleted worker pod is deregistered, and its bound actors are marked CRASHED, but their node state is never terminated, so actors/<uid> stays on the node until the actor is deleted or reverted, if it ever is. The worker-delete workflow now sends atelet's Terminate before releasing each bound actor. It is best-effort, so an unreachable node cannot wedge deregistration. Each Terminate gets 30s, so a hung unmount cannot run out the delete's deadline. Once one times out or returns Unavailable, the worker's remaining actors skip that node and are left to their revert or delete. Each release holds the actor's lease, so it never lands inside a suspend. An actor whose lease is taken is left for a retry: DeleteWorker returns ABORTED, which workersync requeues. Terminate now skips workload teardown when the worker is gone (missing socket, no sandbox record, or UNAVAILABLE) while still reclaiming the actor's node state. Fixes #1688
1 parent 872ab20 commit 9aaa731

12 files changed

Lines changed: 913 additions & 94 deletions

‎cmd/ateapi/internal/controlapi/service.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ func NewRPCService(
107107
}
108108
s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, storageClassLister, instruments, egressGatewayAddress, s, objectStorePlugin)
109109
s.actorWorkflow.clusterSnapshotLocation = snapshotLocation
110-
s.workerWorkflow = NewWorkerWorkflow(impl)
110+
s.workerWorkflow = NewWorkerWorkflow(impl, dialer)
111111
return s
112112
}
113113

‎cmd/ateapi/internal/controlapi/worker_assignment_reconciler.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ type WorkerAssignmentReconciler struct {
5757
func NewWorkerAssignmentReconciler(persistence workerWorkflowStore, workers workerWatcher) *WorkerAssignmentReconciler {
5858
return &WorkerAssignmentReconciler{
5959
persistence: persistence,
60-
workflow: NewWorkerWorkflow(persistence),
60+
workflow: NewWorkerWorkflow(persistence, nil),
6161
workers: workers,
6262
queue: workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[string]()),
6363
}

‎cmd/ateapi/internal/controlapi/worker_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ func newWorkerAPIService(t *testing.T) (*RPCService, store.Interface) {
7474
persistence, cleanup := storetest.SetupTestStore(t)
7575
t.Cleanup(cleanup)
7676
impl := newServiceImpl(persistence, nil)
77-
return &RPCService{impl: impl, workerWorkflow: NewWorkerWorkflow(persistence)}, persistence
77+
return &RPCService{impl: impl, workerWorkflow: NewWorkerWorkflow(persistence, nil)}, persistence
7878
}
7979

8080
// seedAPIWorker registers a worker directly through the store and returns it as

‎cmd/ateapi/internal/controlapi/workflow.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -195,12 +195,14 @@ type actorWorkflowStore interface {
195195
// does from the other side: releasing the Actor bound to a Worker stays
196196
// in-process because there is no bind/release RPC.
197197
type WorkerWorkflow struct {
198-
store workerWorkflowStore
198+
store workerWorkflowStore
199+
dialer *AteletDialer
199200
}
200201

201-
// NewWorkerWorkflow creates a new WorkerWorkflow.
202-
func NewWorkerWorkflow(store workerWorkflowStore) *WorkerWorkflow {
203-
return &WorkerWorkflow{store: store}
202+
// NewWorkerWorkflow creates a new WorkerWorkflow. A nil dialer disables the
203+
// node-state reclaim step (tests that assert only on store transitions).
204+
func NewWorkerWorkflow(store workerWorkflowStore, dialer *AteletDialer) *WorkerWorkflow {
205+
return &WorkerWorkflow{store: store, dialer: dialer}
204206
}
205207

206208
// workerWorkflowStore enumerates the exact storage methods needed by

‎cmd/ateapi/internal/controlapi/workflow_delete.go‎

Lines changed: 32 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -174,28 +174,7 @@ func (w *ActorWorkflow) ensureAteletTerminated(ctx context.Context, actorRef res
174174
slog.String("actor", actorRef.Name),
175175
slog.String("templateAtespace", actor.GetActorTemplate().GetAtespace()),
176176
slog.String("templateName", actor.GetActorTemplate().GetName()))
177-
// Given that external volumes has been in the codebase for a while and
178-
// its preview-ness is solely around the API, and that the impact of
179-
// disabling the gate while in use would be an orphaned volume
180-
// attachment, we are NOT checking the ExternalVolumes gate in the
181-
// delete/suspend path.
182-
workloadSpec = &ateletpb.WorkloadSpec{}
183-
if preview.IsEnabled(preview.GateExternalVolumes) {
184-
for _, vol := range actor.GetStatus().GetExternalVolumes().GetVolumes() {
185-
// StorageVolumeId is only populated once the volume is provisioned.
186-
// Skip volumes that were never created (e.g. failed during PENDING state).
187-
if vol.GetStorageVolumeId() != "" {
188-
workloadSpec.Volumes = append(workloadSpec.Volumes, &ateletpb.Volume{
189-
Name: vol.GetName(),
190-
External: &ateletpb.ExternalVolumeSource{
191-
StorageVolumeId: vol.GetStorageVolumeId(),
192-
VolumeType: vol.GetVolumeType(),
193-
VolumeContext: vol.GetVolumeContext(),
194-
},
195-
})
196-
}
197-
}
198-
}
177+
workloadSpec = fallbackWorkloadSpec(actor)
199178
}
200179

201180
req := &ateletpb.TerminateRequest{
@@ -219,6 +198,37 @@ func (w *ActorWorkflow) ensureAteletTerminated(ctx context.Context, actorRef res
219198
return nil
220199
}
221200

201+
// fallbackWorkloadSpec is the spec to terminate an actor with when its template
202+
// cannot be resolved — deleted, or never readable from the caller in the first
203+
// place. Only the external volumes recorded on the actor are carried, because
204+
// they are the one part of the spec a teardown still acts on: atelet unmounts
205+
// them from the node. A volume with no StorageVolumeId was never provisioned
206+
// (e.g. the actor failed while PENDING), so there is nothing mounted to name.
207+
func fallbackWorkloadSpec(actor *ateapipb.Actor) *ateletpb.WorkloadSpec {
208+
// Given that external volumes has been in the codebase for a while and
209+
// its preview-ness is solely around the API, and that the impact of
210+
// disabling the gate while in use would be an orphaned volume
211+
// attachment, we are NOT checking the ExternalVolumes gate in the
212+
// delete/suspend path.
213+
spec := &ateletpb.WorkloadSpec{}
214+
if preview.IsEnabled(preview.GateExternalVolumes) {
215+
for _, vol := range actor.GetStatus().GetExternalVolumes().GetVolumes() {
216+
if vol.GetStorageVolumeId() == "" {
217+
continue
218+
}
219+
spec.Volumes = append(spec.Volumes, &ateletpb.Volume{
220+
Name: vol.GetName(),
221+
External: &ateletpb.ExternalVolumeSource{
222+
StorageVolumeId: vol.GetStorageVolumeId(),
223+
VolumeType: vol.GetVolumeType(),
224+
VolumeContext: vol.GetVolumeContext(),
225+
},
226+
})
227+
}
228+
}
229+
return spec
230+
}
231+
222232
// ensureVolumesDetachedForDelete detaches external volumes.
223233
func (w *ActorWorkflow) ensureVolumesDetachedForDelete(ctx context.Context, actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate) (err error) {
224234
ctx, done := stepSpan(ctx, "DetachVolumesForDelete")

‎cmd/ateapi/internal/controlapi/workflow_delete_test.go‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -410,17 +410,16 @@ func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) {
410410
}
411411
fresh = uri
412412
objects.PutSnapshot(t, fresh, "manifest.json")
413-
// The worker's pod goes away with the commit still outstanding, so
414-
// the suspend never gets to finish.
415-
if _, err := NewWorkerWorkflow(persistence).DeleteWorker(ctx, workerName, store.DeletePreconditions{}); err != nil {
416-
t.Errorf("DeleteWorker: %v", err)
417-
return err
418-
}
419413
return apierror.Unavailable("worker pod is gone")
420414
}
421415
if _, err := actorWorkflow.SuspendActor(ctx, actorRef, suspendSync); apierror.Code(err) != codes.Unavailable {
422416
t.Fatalf("SuspendActor err = %v, want Unavailable from the failed checkpoint", err)
423417
}
418+
// SuspendActor failed mid-checkpoint and released the actor's lease,
419+
// leaving the actor in SUSPENDING; DeleteWorker then crashes it.
420+
if _, err := NewWorkerWorkflow(persistence, nil).DeleteWorker(ctx, workerName, store.DeletePreconditions{}); err != nil {
421+
t.Fatalf("DeleteWorker: %v", err)
422+
}
424423

425424
// The actor is CRASHED; DeleteActor deletes the in-progress snapshot
426425
// (in_progress_snapshot_uri).

‎cmd/ateapi/internal/controlapi/workflow_reconcile_assignments_test.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ func mustRaiseEpoch(t *testing.T, ctx context.Context, svc *RPCService, persiste
6767

6868
func mustReconcileAssignments(t *testing.T, ctx context.Context, persistence store.Interface) {
6969
t.Helper()
70-
if err := NewWorkerWorkflow(persistence).ReconcileAssignments(ctx, apiWorkerName); err != nil {
70+
if err := NewWorkerWorkflow(persistence, nil).ReconcileAssignments(ctx, apiWorkerName); err != nil {
7171
t.Fatalf("ReconcileAssignments() failed: %v", err)
7272
}
7373
}
@@ -377,7 +377,7 @@ func TestReconcileAssignments_WaitsForBusyActor(t *testing.T) {
377377
if err != nil {
378378
t.Fatalf("AcquireLease() failed: %v", err)
379379
}
380-
err = NewWorkerWorkflow(persistence).ReconcileAssignments(ctx, apiWorkerName)
380+
err = NewWorkerWorkflow(persistence, nil).ReconcileAssignments(ctx, apiWorkerName)
381381
if !errors.Is(err, errActorBusy) {
382382
t.Fatalf("ReconcileAssignments() = %v, want errActorBusy", err)
383383
}
@@ -454,7 +454,7 @@ func TestReconcileAssignments_KeepsActorReboundSinceList(t *testing.T) {
454454
t.Fatalf("UpdateActor() failed: %v", err)
455455
}
456456
}}
457-
if err := NewWorkerWorkflow(rebinding).ReconcileAssignments(ctx, apiWorkerName); err != nil {
457+
if err := NewWorkerWorkflow(rebinding, nil).ReconcileAssignments(ctx, apiWorkerName); err != nil {
458458
t.Fatalf("ReconcileAssignments() failed: %v", err)
459459
}
460460

@@ -483,7 +483,7 @@ func TestReconcileAssignments_RetriesRecordAfterConcurrentBind(t *testing.T) {
483483
binding := &hookedStore{Interface: persistence, beforeUpdateWorker: func() {
484484
assignAPIWorker(t, ctx, persistence, apiWorkerName, boundUID)
485485
}}
486-
if err := NewWorkerWorkflow(binding).ReconcileAssignments(ctx, apiWorkerName); err != nil {
486+
if err := NewWorkerWorkflow(binding, nil).ReconcileAssignments(ctx, apiWorkerName); err != nil {
487487
t.Fatalf("ReconcileAssignments() failed: %v", err)
488488
}
489489

@@ -517,7 +517,7 @@ func TestReconcileAssignments_LeavesReplacedWorker(t *testing.T) {
517517
}
518518
seedEpochWorker(t, ctx, persistence, 1, 1)
519519
}}
520-
if err := NewWorkerWorkflow(replacing).ReconcileAssignments(ctx, apiWorkerName); err != nil {
520+
if err := NewWorkerWorkflow(replacing, nil).ReconcileAssignments(ctx, apiWorkerName); err != nil {
521521
t.Fatalf("ReconcileAssignments() failed: %v", err)
522522
}
523523

@@ -535,7 +535,7 @@ func TestReconcileAssignments_FailureKeepsObservedEpoch(t *testing.T) {
535535
seedRunningActor(t, ctx, persistence)
536536
mustRaiseEpoch(t, ctx, svc, persistence, 2)
537537

538-
failing := NewWorkerWorkflow(failingUpdateActorStore{Interface: persistence, err: errors.New("crash failed")})
538+
failing := NewWorkerWorkflow(failingUpdateActorStore{Interface: persistence, err: errors.New("crash failed")}, nil)
539539
if err := failing.ReconcileAssignments(ctx, apiWorkerName); err == nil {
540540
t.Fatal("ReconcileAssignments() = nil error, want the crash failure reported")
541541
}

‎cmd/ateapi/internal/controlapi/workflow_testutil_test.go‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -367,13 +367,19 @@ func mustReadActor(t *testing.T, ctx context.Context, st store.Interface, actorR
367367
}
368368

369369
// workerDeletedDuringReleaseStore runs the DeleteWorker workflow's release of
370-
// an actor right after a finalize step frees the actor's worker. That workflow
371-
// takes no actor lease, so its crash can land between the step's read of the
372-
// actor and its write.
370+
// an actor right after a finalize step frees the actor's worker, so its crash
371+
// lands between the step's read of the actor and its write. In production the
372+
// release takes the actor's lease, which pause and suspend hold through their
373+
// finalize steps, so it waits for them instead; AcquireLease is bypassed here
374+
// to keep the finalize step's own version guard against the crash covered.
373375
type workerDeletedDuringReleaseStore struct {
374376
store.Interface
375377
}
376378

379+
func (s workerDeletedDuringReleaseStore) AcquireLease(ctx context.Context, _ string) (*store.Lease, error) {
380+
return store.NewLease(ctx, func() {}), nil
381+
}
382+
377383
func (s workerDeletedDuringReleaseStore) ReleaseActorFromWorker(ctx context.Context, workerName, actorUID string) (*ateapipb.Worker, error) {
378384
worker, err := s.Interface.GetWorker(ctx, workerName)
379385
if err != nil {
@@ -387,7 +393,7 @@ func (s workerDeletedDuringReleaseStore) ReleaseActorFromWorker(ctx context.Cont
387393
if err != nil {
388394
return nil, err
389395
}
390-
if err := NewWorkerWorkflow(s.Interface).releaseBoundActor(ctx, worker, assignment); err != nil {
396+
if err := NewWorkerWorkflow(s, nil).releaseBoundActor(ctx, worker, assignment, &nodeReclaim{}); err != nil {
391397
return nil, err
392398
}
393399
return released, nil

0 commit comments

Comments
 (0)