Skip to content

Commit d083fd3

Browse files
committed
ateapi: validate a crashed suspend's report before ignoring it
A report naming a crashed suspend's snapshot skipped the uuid, upload URI and fidelity checks in applyLatestDurable. Move the check into applyLatestDurable after those checks, so a CRASHED actor goes through applyActorSnapshotState like a SUSPENDED or RUNNING one. Check the actor's state before cloning it.
1 parent 41b4d6e commit d083fd3

2 files changed

Lines changed: 34 additions & 33 deletions

File tree

‎cmd/ateapi/internal/controlapi/snapshot_report.go‎

Lines changed: 20 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -57,41 +57,21 @@ func (w *ActorWorkflow) ApplyReportedSnapshotState(ctx context.Context, actorRef
5757
return err
5858
}
5959

60-
applied := proto.Clone(before).(*ateapi.Actor)
61-
status := applied.GetStatus()
6260
// A workflow owns a transitioning Actor's snapshots, even after its lease
6361
// lapses.
64-
switch st := status.GetState(); st {
65-
case ateapi.ActorState_ACTOR_STATE_SUSPENDED, ateapi.ActorState_ACTOR_STATE_RUNNING:
66-
if err := applyActorSnapshotState(leaseCtx, status, node, state); err != nil {
67-
return err
68-
}
69-
case ateapi.ActorState_ACTOR_STATE_CRASHED:
70-
reported := state.GetLatestDurable()
71-
snap := findSnapshotByGeneration(status.GetSnapshots(), reported.GetGeneration())
72-
// A crashed Actor keeps its AssignedNode, so only Locality tells a crashed
73-
// suspend's snapshot from an upload that finished on node: only a finalized
74-
// suspend sets Locality. Its upload is not recorded, and a revert deletes
75-
// it with the rest of the discarded execution.
76-
if snap != nil && snap.GetLocality() == "" && snap.GetDurableSnapshot().GetStatus() != ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED {
77-
slog.InfoContext(leaseCtx, msgDurableCrashedSuspend,
78-
slog.Int("generation", int(reported.GetGeneration())),
79-
slog.String("uuid", reported.GetUuid()))
80-
} else if err := applyLatestDurable(leaseCtx, status, reported); err != nil {
81-
return err
82-
}
83-
if state.GetLatestLocal() == nil && state.GetUploading() == nil {
84-
if err := applyDetach(status, node, reported.GetGeneration()); err != nil {
85-
return err
86-
}
87-
}
62+
switch st := before.GetStatus().GetState(); st {
63+
case ateapi.ActorState_ACTOR_STATE_SUSPENDED, ateapi.ActorState_ACTOR_STATE_RUNNING, ateapi.ActorState_ACTOR_STATE_CRASHED:
8864
case ateapi.ActorState_ACTOR_STATE_SUSPENDING, ateapi.ActorState_ACTOR_STATE_RESUMING,
8965
ateapi.ActorState_ACTOR_STATE_DELETING, ateapi.ActorState_ACTOR_STATE_REVERTING:
9066
return apierror.Aborted("actor %s is %s; retry once that finishes", actorRef, st)
9167
default:
9268
return apierror.Internal("actor %s has unexpected state %s", actorRef, st)
9369
}
9470

71+
applied := proto.Clone(before).(*ateapi.Actor)
72+
if err := applyActorSnapshotState(leaseCtx, applied.GetStatus(), node, state); err != nil {
73+
return err
74+
}
9575
if proto.Equal(applied.GetStatus(), before.GetStatus()) {
9676
return nil
9777
}
@@ -177,8 +157,10 @@ func applyAteletResponseState(ctx context.Context, status *ateapi.ActorStatus, s
177157
//
178158
// A late report changes nothing and is logged at Info. A report is late when
179159
// its generation is older than the latest COMPLETED one, or is no longer in
180-
// status.snapshots, for example because a revert removed it. A repeat of the
181-
// latest COMPLETED generation also changes nothing, and isn't logged.
160+
// status.snapshots, for example because a revert removed it. A CRASHED
161+
// Actor's report of a snapshot whose suspend crashed is logged the same way. A
162+
// repeat of the latest COMPLETED generation also changes nothing, and isn't
163+
// logged.
182164
func applyLatestDurable(ctx context.Context, status *ateapi.ActorStatus, reported *ateapi.SnapshotReport) error {
183165
if reported == nil {
184166
return nil
@@ -215,6 +197,16 @@ func applyLatestDurable(ctx context.Context, status *ateapi.ActorStatus, reporte
215197
if reported.GetFidelity() == ateapi.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED {
216198
return apierror.InvalidArgument("snapshot generation %d reports no fidelity", reported.GetGeneration())
217199
}
200+
// A crashed Actor keeps its AssignedNode, so only Locality tells a crashed
201+
// suspend's snapshot from an upload that finished on node: only a finalized
202+
// suspend sets Locality. Its upload is not recorded, and a revert deletes
203+
// it with the rest of the discarded execution.
204+
if status.GetState() == ateapi.ActorState_ACTOR_STATE_CRASHED && snap.GetLocality() == "" {
205+
slog.InfoContext(ctx, msgDurableCrashedSuspend,
206+
slog.Int("generation", int(reported.GetGeneration())),
207+
slog.String("uuid", reported.GetUuid()))
208+
return nil
209+
}
218210
snap.Fidelity = reported.GetFidelity()
219211
snap.DurableSnapshot.Status = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED
220212
return nil

‎cmd/ateapi/internal/controlapi/snapshot_report_test.go‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -319,10 +319,10 @@ func TestApplyActorSnapshotState(t *testing.T) {
319319
}
320320
}
321321

322-
// A reported durable snapshot older than the latest COMPLETED one, or at a
323-
// generation status no longer has, is logged at Info, once, with the
324-
// generation it named. A repeat of the latest, and one that is applied, aren't
325-
// logged.
322+
// A reported durable snapshot older than the latest COMPLETED one, at a
323+
// generation status no longer has, or of a CRASHED actor's crashed suspend, is
324+
// logged at Info, once, with the generation it named. A repeat of the latest,
325+
// and one that is applied, aren't logged.
326326
func TestApplyLatestDurableLogsStaleReports(t *testing.T) {
327327
const (
328328
pending = ateapi.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING
@@ -351,6 +351,15 @@ func TestApplyLatestDurableLogsStaleReports(t *testing.T) {
351351
gen: 2,
352352
wantMsg: msgDurableNotRecorded,
353353
wantAttrs: map[string]string{"generation": "2", "uuid": snapName(2)},
354+
}, {
355+
name: "a CRASHED actor's crashed suspend",
356+
status: &ateapi.ActorStatus{State: ateapi.ActorState_ACTOR_STATE_CRASHED, LastAssignedGeneration: 2, Snapshots: []*ateapi.Snapshot{
357+
withDurable(t, 1, reportNode, completed),
358+
withDurable(t, 2, "", pending),
359+
}},
360+
gen: 2,
361+
wantMsg: msgDurableCrashedSuspend,
362+
wantAttrs: map[string]string{"generation": "2", "uuid": snapName(2)},
354363
}, {
355364
name: "the latest COMPLETED generation itself",
356365
status: &ateapi.ActorStatus{LastAssignedGeneration: 2, Snapshots: []*ateapi.Snapshot{
@@ -373,7 +382,7 @@ func TestApplyLatestDurableLogsStaleReports(t *testing.T) {
373382
var got []logged
374383
prev := slog.Default()
375384
slog.SetDefault(slog.New(slogHandlerFunc(func(r slog.Record) {
376-
if r.Message != msgDurableOlder && r.Message != msgDurableNotRecorded {
385+
if r.Message != msgDurableOlder && r.Message != msgDurableNotRecorded && r.Message != msgDurableCrashedSuspend {
377386
return
378387
}
379388
l := logged{msg: r.Message, attrs: map[string]string{}}

0 commit comments

Comments
 (0)