Skip to content

Commit 27f9b07

Browse files
adalundheclaude
andcommitted
fix(session): recover a runtime that crashed during suspension
Why: when a worker is lost while the Actor suspends after a turn, for example during a node drain or scale-down, the Actor ends CRASHED and the Session still reads READY. The idle worker keeps the suspension's claim because the outcome is unknown, and nothing takes the claim over on a timeout. Every later send waits out the 10-second busy period and is told to retry, so the conversation never recovers, even with automatic recovery on. Only a crash recovery retires the claim. Reservation now reports a held suspension claim as ErrDispatchSuspending, a kind of ErrDispatchBusy. requireSettledRuntime's other callers still see ErrFailedPrecondition. With automatic recovery on, a send whose wait ends on that claim, which a healthy suspension does not reach, attempts the bounded recovery once. It then reserves again without waiting. Clients still get ErrSendNotAccepted, so with the flag off nothing changes. Reproduced on kind before and after: the worker was deleted while the Actor was SUSPENDING. Before the fix, every send waited 10 seconds and was refused, still stuck after six minutes. After it, the first send recovered the Session, including one left stuck from the earlier run, and was answered. Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Ada Lundhe <[email protected]>
1 parent 01f14d6 commit 27f9b07

5 files changed

Lines changed: 198 additions & 15 deletions

File tree

‎docs/architecture/runtime-and-lifecycle.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,18 @@ runtime is restored and the send proceeds, and a busy healthy runtime refuses it
282282
as before. Other conflicts, such as a checkpoint in progress, do not start
283283
recovery. A send while recovery is pending joins it.
284284

285+
A worker can also be lost while the Actor suspends after a turn, for example
286+
during a node drain or scale-down. The Actor is then CRASHED, the Session still
287+
reads READY, and the idle worker keeps the suspension's claim because the outcome
288+
is unknown; nothing takes the claim over on a timeout. Every later send waits out
289+
the 10-second busy period and is told to retry, so the conversation would never
290+
recover on its own. Reservation reports a held suspension claim as
291+
`ErrDispatchSuspending`, a kind of `ErrDispatchBusy`. When that claim outlasts
292+
the whole wait, which a healthy suspension does not, the send attempts the same
293+
bounded recovery once. Admitting the recovery retires the claim. Revert leaves a
294+
suspension that is still running alone, and one more reservation, without
295+
another wait, proceeds or answers retry as before.
296+
285297
A send waits at most 10 seconds on the recovery it starts. A recovery still
286298
running then stays admitted: the attempt releases its claim, the send is
287299
answered `KAGENT_SEND_NOT_ACCEPTED` with `state: RECOVERING` and a one-second

‎go/core/internal/database/session_recovery_test.go‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,42 @@ func TestCrashedRecoveryAdmissionRespectsFences(t *testing.T) {
142142
// A crash during post-turn suspension leaves the worker's claim issued, because
143143
// the worker cannot tell whether its suspension took effect. The confirmed crash
144144
// settles that: admission retires the claim instead of waiting for it.
145+
// A suspension whose worker was lost keeps its claim. Reservation names that
146+
// cause so a send can recover the Session; a crash recovery retires the claim.
147+
func TestDispatchReportsUnsettledSuspension(t *testing.T) {
148+
client := NewClient(setupTestDB(t), "public")
149+
sessionFixture(t, client, t.Context(), "team-a", "revision", "assistant", "kagent")
150+
session, task := waitingTaskFixture(t, client)
151+
_, version, err := client.GetVersionedSessionTask(t.Context(), session.Id, string(task.ID))
152+
require.NoError(t, err)
153+
dispatch := uuid.New()
154+
require.NoError(t, client.ReserveSessionDispatch(t.Context(), session.Id, dispatch, ""))
155+
busy := client.ReserveSessionDispatch(t.Context(), session.Id, uuid.New(), "")
156+
require.ErrorIs(t, busy, ErrDispatchBusy)
157+
require.NotErrorIs(t, busy, ErrDispatchSuspending, "an ordinary dispatch in flight is not a suspension")
158+
revoked, err := client.RevokeSessionDispatch(t.Context(), session.Id, dispatch, "unused")
159+
require.NoError(t, err)
160+
require.True(t, revoked)
161+
162+
for _, state := range []a2a.TaskState{a2a.TaskStateWorking, a2a.TaskStateInputRequired} {
163+
task.Status.State = state
164+
version, err = client.UpdateSessionTask(t.Context(), session.Id, version, taskMutationHash(string(state)), task, task, "")
165+
require.NoError(t, err)
166+
}
167+
require.NoError(t, client.SettleSessionTask(t.Context(), session.Id, string(task.ID), version))
168+
_, err = client.ClaimSessionQuiescence(t.Context())
169+
require.NoError(t, err)
170+
suspending := client.ReserveSessionDispatch(t.Context(), session.Id, uuid.New(), "")
171+
require.ErrorIs(t, suspending, ErrDispatchSuspending)
172+
require.ErrorIs(t, suspending, ErrDispatchBusy, "callers that only wait keep treating it as busy")
173+
174+
uid := "actor-" + session.Id
175+
operation, err := client.BeginCrashedSessionRecovery(t.Context(), observedGeneration(t, client, session.Id, uid), SessionTaskSnapshot{})
176+
require.NoError(t, err)
177+
finishCrashedRecovery(t, client, session.Id, operation, uid)
178+
require.NoError(t, client.ReserveSessionDispatch(t.Context(), session.Id, uuid.New(), ""), "recovery retires the claim")
179+
}
180+
145181
func TestCrashedRecoveryRetiresIssuedIdleSuspension(t *testing.T) {
146182
client := NewClient(setupTestDB(t), "public")
147183
sessionFixture(t, client, t.Context(), "team-a", "revision", "assistant", "kagent")

‎go/core/internal/database/session_tasks.go‎

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,18 @@ import (
2323
// or idle lifecycle operation currently owns the session.
2424
var ErrDispatchBusy = errors.New("session temporarily unavailable for dispatch")
2525

26+
// ErrDispatchSuspending is ErrDispatchBusy because a post-turn suspension holds its
27+
// claim. The suspension is running, or its worker was lost and its outcome is
28+
// unknown: the idle worker keeps the claim rather than guess, and nothing takes it
29+
// over on a timeout. Only a crash recovery or Delete retires it, so the send path
30+
// needs to tell this cause apart from an ordinary busy dispatch.
31+
var ErrDispatchSuspending = fmt.Errorf("runtime suspension has not settled: %w", ErrDispatchBusy)
32+
33+
// errSuspensionUnsettled is requireSettledRuntime's failure when a claimed
34+
// suspension is what keeps the runtime unsettled. It still matches
35+
// ErrFailedPrecondition for every other caller.
36+
var errSuspensionUnsettled = fmt.Errorf("runtime suspension has not settled: %w", ErrFailedPrecondition)
37+
2638
// ErrMessageAccepted means an earlier dispatch already persisted this initial input.
2739
var ErrMessageAccepted = errors.New("message already accepted")
2840

@@ -70,6 +82,9 @@ func (c *Client) ReserveSessionDispatch(ctx context.Context, sessionID string, d
7082
return ErrConflict
7183
}
7284
if err := requireSettledRuntime(ctx, tx, session.HistoryID, ""); err != nil {
85+
if errors.Is(err, errSuspensionUnsettled) {
86+
return ErrDispatchSuspending
87+
}
7388
if errors.Is(err, ErrFailedPrecondition) {
7489
return ErrDispatchBusy
7590
}
@@ -829,16 +844,23 @@ func readTaskMessages(ctx context.Context, db dbExecutor, historyID uuid.UUID, t
829844
// another unexpired dispatch. A writer may pass its own dispatch ID.
830845
// Callers hold the session lock so new execution and lifecycle claims cannot race.
831846
func requireSettledRuntime(ctx context.Context, db dbExecutor, historyID uuid.UUID, dispatchID string) error {
832-
pending, err := queryOne(ctx, db, `
847+
type unsettled struct {
848+
Suspending, Pending bool
849+
}
850+
state, err := queryOne(ctx, db, `
833851
SELECT EXISTS (SELECT 1 FROM session_task_event WHERE history_id = $1
834-
AND (NOT published OR (quiescence_pending AND quiescence_executor_id IS NOT NULL)))
852+
AND quiescence_pending AND quiescence_executor_id IS NOT NULL) AS suspending,
853+
EXISTS (SELECT 1 FROM session_task_event WHERE history_id = $1 AND NOT published)
835854
OR EXISTS (SELECT 1 FROM session WHERE history_id = $1 AND dispatch_expires_at > clock_timestamp()
836-
AND ($2::text = '' OR dispatch_id::text <> $2))
837-
`, pgx.RowTo[bool], historyID, dispatchID)
855+
AND ($2::text = '' OR dispatch_id::text <> $2)) AS pending
856+
`, pgx.RowToStructByName[unsettled], historyID, dispatchID)
838857
if err != nil {
839858
return err
840859
}
841-
if pending {
860+
if state.Suspending {
861+
return errSuspensionUnsettled
862+
}
863+
if state.Pending {
842864
return fmt.Errorf("runtime cleanup or lifecycle work is not yet settled: %w", ErrFailedPrecondition)
843865
}
844866
return nil

‎go/core/internal/service/session/interactions.go‎

Lines changed: 43 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ type InteractionService struct {
5555
automaticRecovery bool
5656
// recoveryWait bounds how long a send waits on the recovery it starts.
5757
recoveryWait time.Duration
58+
// dispatchWait bounds how long a send waits for a busy Session.
59+
dispatchWait time.Duration
5860
}
5961

6062
// sendRecoveryWait bounds a send's wait for crash recovery. A recovery still
@@ -63,7 +65,8 @@ type InteractionService struct {
6365
const sendRecoveryWait = 10 * time.Second
6466

6567
func NewInteractionService(store interactionStore, agents agentService, sessions *Service, allowPushHTTP, automaticRecovery bool) *InteractionService {
66-
return &InteractionService{store: store, agents: agents, sessions: sessions, allowPushHTTP: allowPushHTTP, automaticRecovery: automaticRecovery, recoveryWait: sendRecoveryWait}
68+
return &InteractionService{store: store, agents: agents, sessions: sessions, allowPushHTTP: allowPushHTTP, automaticRecovery: automaticRecovery,
69+
recoveryWait: sendRecoveryWait, dispatchWait: 10 * time.Second}
6770
}
6871

6972
// PreparedSend either grants a new dispatch or returns an already accepted task.
@@ -170,19 +173,40 @@ func (s *InteractionService) PrepareSend(ctx context.Context, agent types.Namesp
170173
config.PushConfig = nil
171174
req.Config = &config
172175
}
173-
id, err := s.reserveDispatch(ctx, session.Id, initialID)
176+
id, err := s.reserveDispatch(ctx, session.Id, initialID, s.dispatchWait)
177+
if s.automaticRecovery && errors.Is(err, errSuspensionUnsettled) {
178+
// Why: a worker lost while the Actor suspends after a turn leaves the
179+
// Actor CRASHED, the Session READY, and the suspension's claim held, since
180+
// its outcome is unknown and nothing takes the claim over on a timeout.
181+
// Every later send then waits out the busy period and is told to retry,
182+
// so the conversation never recovers on its own. A crash recovery is the
183+
// only thing that retires that claim. Reaching here means the claim
184+
// outlasted the whole wait, which a healthy suspension does not, so the
185+
// send tries the bounded recovery once. Revert leaves a still-running
186+
// suspension alone, and one more reservation, without another wait,
187+
// proceeds or answers retry as before.
188+
switch recoverErr := s.recoverForSend(ctx, session.Id); {
189+
case recoverErr == nil:
190+
id, err = s.reserveDispatch(ctx, session.Id, initialID, 0)
191+
case !errors.Is(recoverErr, errRecoveryUnavailable):
192+
return nil, recoverErr
193+
}
194+
}
174195
if s.automaticRecovery && errors.Is(err, database.ErrActiveTurn) {
175196
// A crash during a turn leaves that turn open, and an open turn refuses
176197
// every send, so no send would reach the crashed Actor to recover it.
177198
// Revert restores a crashed Actor and leaves a busy one alone, so one more
178199
// reservation either proceeds or is refused as before.
179200
switch recoverErr := s.recoverForSend(ctx, session.Id); {
180201
case recoverErr == nil:
181-
id, err = s.reserveDispatch(ctx, session.Id, initialID)
202+
id, err = s.reserveDispatch(ctx, session.Id, initialID, s.dispatchWait)
182203
case !errors.Is(recoverErr, errRecoveryUnavailable):
183204
return nil, recoverErr
184205
}
185206
}
207+
if errors.Is(err, errSuspensionUnsettled) {
208+
err = ErrSendNotAccepted
209+
}
186210
switch {
187211
case errors.Is(err, database.ErrActiveTurn):
188212
err = errTurnActive
@@ -239,10 +263,11 @@ func (s *InteractionService) prepareTask(ctx context.Context, agent types.Namesp
239263

240264
// reserveDispatch waits only before forwarding input. No runtime send is retried
241265
// on an ambiguous transport error, and no SQL lock is held during dispatch.
242-
func (s *InteractionService) reserveDispatch(ctx context.Context, sessionID, initialID string) (uuid.UUID, error) {
266+
// It waits at most wait for a busy Session, answering ErrSendNotAccepted after
267+
// it, or errSuspensionUnsettled when a claimed suspension was still the cause.
268+
func (s *InteractionService) reserveDispatch(ctx context.Context, sessionID, initialID string, wait time.Duration) (uuid.UUID, error) {
243269
id := uuid.New()
244-
deadline := time.NewTimer(10 * time.Second)
245-
defer deadline.Stop()
270+
deadline := time.Now().Add(wait)
246271
for {
247272
err := s.store.ReserveSessionDispatch(ctx, sessionID, id, initialID)
248273
if err == nil {
@@ -254,19 +279,27 @@ func (s *InteractionService) reserveDispatch(ctx context.Context, sessionID, ini
254279
if !errors.Is(err, database.ErrDispatchBusy) {
255280
return uuid.Nil, interactionStoreError(ctx, err)
256281
}
257-
timer := time.NewTimer(100 * time.Millisecond)
282+
if !time.Now().Before(deadline) {
283+
if errors.Is(err, database.ErrDispatchSuspending) {
284+
return uuid.Nil, errSuspensionUnsettled
285+
}
286+
return uuid.Nil, ErrSendNotAccepted
287+
}
288+
timer := time.NewTimer(min(100*time.Millisecond, time.Until(deadline)))
258289
select {
259290
case <-ctx.Done():
260291
timer.Stop()
261292
return uuid.Nil, ctx.Err()
262-
case <-deadline.C:
263-
timer.Stop()
264-
return uuid.Nil, ErrSendNotAccepted
265293
case <-timer.C:
266294
}
267295
}
268296
}
269297

298+
// errSuspensionUnsettled is reserveDispatch's answer when a claimed post-turn
299+
// suspension still held the Session at the end of the wait. PrepareSend recovers
300+
// it or answers ErrSendNotAccepted, so it never reaches a client.
301+
var errSuspensionUnsettled = errors.New("runtime suspension has not settled")
302+
270303
// errTurnActive refuses a send while an earlier turn is still open. A runtime
271304
// that crashed mid-turn leaves its turn open, so unless the send recovers it,
272305
// this is how the crash reaches the client. The reason lets a client offer

‎go/core/internal/service/session/recovery_test.go‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -692,3 +692,83 @@ func TestRecoverRejectedSendReturnsSessionErrors(t *testing.T) {
692692
})
693693
}
694694
}
695+
696+
// suspensionTestStore answers reservations with reserveErr until a revert clears it.
697+
type suspensionTestStore struct {
698+
*database.Client
699+
reserveErr error
700+
reserves int
701+
}
702+
703+
func (s *suspensionTestStore) ReserveSessionDispatch(context.Context, string, uuid.UUID, string) error {
704+
s.reserves++
705+
return s.reserveErr
706+
}
707+
708+
// suspensionTestWorkflow reverts a crashed runtime by retiring the held claim.
709+
type suspensionTestWorkflow struct {
710+
serviceTestWorkflow
711+
store *suspensionTestStore
712+
crashed bool
713+
reverts int
714+
}
715+
716+
func (w *suspensionTestWorkflow) Revert(_ context.Context, session *apiv1.Session) (*apiv1.Session, error) {
717+
w.reverts++
718+
if w.err != nil {
719+
return nil, w.err
720+
}
721+
if w.crashed {
722+
w.store.reserveErr = nil
723+
}
724+
return session, nil
725+
}
726+
727+
// A worker lost during the post-turn suspension leaves the Actor CRASHED and the
728+
// suspension's claim held, so every send waits and is told to retry. Only a send
729+
// that outlasted the wait on that claim, with automatic recovery on, recovers it.
730+
func TestSendRecoversACrashDuringSuspension(t *testing.T) {
731+
for _, test := range []struct {
732+
name string
733+
disabled bool
734+
reserveErr error
735+
crashed bool
736+
revertErr error
737+
reverts int
738+
dispatched bool
739+
}{
740+
{name: "a crash during suspension", reserveErr: database.ErrDispatchSuspending, crashed: true, reverts: 1, dispatched: true},
741+
{name: "a suspension still running", reserveErr: database.ErrDispatchSuspending, reverts: 1},
742+
{name: "a revert that cannot run", reserveErr: database.ErrDispatchSuspending, crashed: true,
743+
revertErr: status.Error(codes.Unavailable, "control plane unavailable"), reverts: 1},
744+
{name: "automatic recovery off", disabled: true, reserveErr: database.ErrDispatchSuspending, crashed: true},
745+
{name: "an ordinary dispatch in flight", reserveErr: database.ErrDispatchBusy, crashed: true},
746+
} {
747+
t.Run("Given "+test.name+", when a send outlasts the busy wait, assert that it recovers only a crashed suspension", func(t *testing.T) {
748+
id := uuid.NewString()
749+
agent := types.NamespacedName{Namespace: "team-a", Name: "assistant"}
750+
stored := &apiv1.Session{Id: id, ContextId: id, State: apiv1.RuntimeState_RUNTIME_STATE_READY,
751+
Operation: apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE,
752+
Agent: &apiv1.AgentProvenance{Ref: &apiv1.ResourceReference{Namespace: agent.Namespace, Name: agent.Name}}}
753+
store := &suspensionTestStore{reserveErr: test.reserveErr}
754+
workflow := &suspensionTestWorkflow{serviceTestWorkflow: serviceTestWorkflow{err: test.revertErr}, store: store, crashed: test.crashed}
755+
interactions := NewInteractionService(store, nil, NewService(&serviceTestStore{getResult: stored}, serviceTestAuthorizer{}, workflow), false, !test.disabled)
756+
const wait = 200 * time.Millisecond
757+
interactions.dispatchWait = wait
758+
message := a2a.NewMessage(a2a.MessageRoleUser, a2a.NewTextPart("What is 2+2?"))
759+
message.ContextID = id
760+
started := time.Now()
761+
prepared, err := interactions.PrepareSend(serviceTestContext("alice"), agent, &a2a.SendMessageRequest{Message: message})
762+
elapsed := time.Since(started)
763+
require.Equal(t, test.reverts, workflow.reverts)
764+
if test.dispatched {
765+
require.NoError(t, err)
766+
require.NotEqual(t, uuid.Nil, prepared.DispatchID)
767+
return
768+
}
769+
require.ErrorIs(t, err, ErrSendNotAccepted, "the client still sees an ordinary retryable refusal")
770+
require.GreaterOrEqual(t, elapsed, wait, "a send first waits out the busy period")
771+
require.Less(t, elapsed, 2*wait, "the reservation after a recovery attempt does not wait again")
772+
})
773+
}
774+
}

0 commit comments

Comments
 (0)