Skip to content

Commit 7315c60

Browse files
fix(server): a failed Claude wake dispatch no longer drops every later wake (pingdotgg#28173)
1 parent 6583670 commit 7315c60

4 files changed

Lines changed: 214 additions & 6 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13766,6 +13766,161 @@ describe("ClaudeAdapterV2 background wake turns", () => {
1376613766
),
1376713767
);
1376813768

13769+
it.effect("offers a later wake after an earlier wake's continuation dispatch failed", () =>
13770+
Effect.scoped(
13771+
Effect.gen(function* () {
13772+
const SECOND_TASK_ID = "b2ndwake";
13773+
const SECOND_TOOL_USE_ID = "toolu_second_wake";
13774+
const SECOND_SUMMARY = 'Background command "Second sleep" completed (exit code 0)';
13775+
const harness = yield* makeWakeHarness;
13776+
const now = yield* DateTime.now;
13777+
13778+
yield* harness.runtime.startTurn(
13779+
makeClaudeTestTurnInput({
13780+
threadId: harness.threadId,
13781+
providerThread: harness.providerThread,
13782+
now,
13783+
attemptId: RunAttemptId.make("attempt-claude-wake-dispatch-failed-1"),
13784+
text: "Run two builds in the background.",
13785+
attachments: [],
13786+
}),
13787+
);
13788+
yield* Queue.offer(harness.sdkMessages, wakeTaskStarted);
13789+
yield* Queue.offer(
13790+
harness.sdkMessages,
13791+
claudeSdkFrame({
13792+
...wakeTaskStarted,
13793+
task_id: SECOND_TASK_ID,
13794+
tool_use_id: SECOND_TOOL_USE_ID,
13795+
description: "Second sleep",
13796+
uuid: "00000000-0000-4000-8000-000000000121",
13797+
}),
13798+
);
13799+
yield* Queue.offer(harness.sdkMessages, turnOneResult);
13800+
yield* harness.awaitTerminal(1);
13801+
13802+
yield* harness.offerAndWait(wakeNotification);
13803+
yield* harness.offerAndWait(wakeTurnInit);
13804+
assert.lengthOf(harness.continuationRequests, 1);
13805+
yield* harness.offerAndWait(wakeAssistant);
13806+
yield* harness.offerAndWait(wakeResult);
13807+
13808+
// A failed dispatch never attaches the wake turn; clearIfCurrent is the only release.
13809+
const failed = harness.continuationRequests[0]!;
13810+
assert.isDefined(failed.clearIfCurrent);
13811+
yield* failed.clearIfCurrent!();
13812+
13813+
yield* harness.offerAndWait(
13814+
claudeSdkFrame({
13815+
...wakeNotification,
13816+
task_id: SECOND_TASK_ID,
13817+
tool_use_id: SECOND_TOOL_USE_ID,
13818+
output_file: `/tmp/claude-replay/tasks/${SECOND_TASK_ID}.output`,
13819+
summary: SECOND_SUMMARY,
13820+
uuid: "00000000-0000-4000-8000-000000000123",
13821+
}),
13822+
);
13823+
yield* harness.offerAndWait(
13824+
claudeSdkFrame({ ...wakeTurnInit, uuid: "00000000-0000-4000-8000-000000000124" }),
13825+
);
13826+
assert.lengthOf(harness.continuationRequests, 2);
13827+
}).pipe(
13828+
Effect.provide(
13829+
Layer.mergeAll(
13830+
IdAllocator.layer,
13831+
McpProviderSessions.layer,
13832+
NodeServices.layer,
13833+
NodeCrypto.layer,
13834+
),
13835+
),
13836+
),
13837+
),
13838+
);
13839+
13840+
it.effect("keeps a newer wake latched when a stale continuation request is cleared", () =>
13841+
Effect.scoped(
13842+
Effect.gen(function* () {
13843+
const SECOND_TASK_ID = "b2ndwake";
13844+
const SECOND_TOOL_USE_ID = "toolu_second_wake";
13845+
const harness = yield* makeWakeHarness;
13846+
const now = yield* DateTime.now;
13847+
13848+
yield* harness.runtime.startTurn(
13849+
makeClaudeTestTurnInput({
13850+
threadId: harness.threadId,
13851+
providerThread: harness.providerThread,
13852+
now,
13853+
attemptId: RunAttemptId.make("attempt-claude-wake-stale-clear-1"),
13854+
text: "Run two builds in the background.",
13855+
attachments: [],
13856+
}),
13857+
);
13858+
yield* Queue.offer(harness.sdkMessages, wakeTaskStarted);
13859+
yield* Queue.offer(
13860+
harness.sdkMessages,
13861+
claudeSdkFrame({
13862+
...wakeTaskStarted,
13863+
task_id: SECOND_TASK_ID,
13864+
tool_use_id: SECOND_TOOL_USE_ID,
13865+
description: "Second sleep",
13866+
uuid: "00000000-0000-4000-8000-000000000131",
13867+
}),
13868+
);
13869+
yield* Queue.offer(harness.sdkMessages, turnOneResult);
13870+
yield* harness.awaitTerminal(1);
13871+
13872+
yield* harness.offerAndWait(wakeNotification);
13873+
yield* harness.offerAndWait(wakeTurnInit);
13874+
assert.lengthOf(harness.continuationRequests, 1);
13875+
yield* harness.runtime.startTurn(
13876+
makeClaudeTestTurnInput({
13877+
threadId: harness.threadId,
13878+
providerThread: harness.providerThread,
13879+
now,
13880+
attemptId: RunAttemptId.make("attempt-claude-wake-stale-clear-2"),
13881+
text: "Background task completed.",
13882+
attachments: [],
13883+
providerTurnOrdinal: 2,
13884+
messageCreatedBy: "agent",
13885+
messageCreationSource: "provider",
13886+
}),
13887+
);
13888+
yield* harness.offerAndWait(wakeAssistant);
13889+
yield* harness.offerAndWait(wakeResult);
13890+
yield* harness.awaitTerminal(2);
13891+
13892+
yield* harness.offerAndWait(
13893+
claudeSdkFrame({
13894+
...wakeNotification,
13895+
task_id: SECOND_TASK_ID,
13896+
tool_use_id: SECOND_TOOL_USE_ID,
13897+
output_file: `/tmp/claude-replay/tasks/${SECOND_TASK_ID}.output`,
13898+
uuid: "00000000-0000-4000-8000-000000000133",
13899+
}),
13900+
);
13901+
yield* harness.offerAndWait(
13902+
claudeSdkFrame({ ...wakeTurnInit, uuid: "00000000-0000-4000-8000-000000000134" }),
13903+
);
13904+
assert.lengthOf(harness.continuationRequests, 2);
13905+
13906+
yield* harness.continuationRequests[0]!.clearIfCurrent!();
13907+
yield* harness.offerAndWait(
13908+
claudeSdkFrame({ ...wakeAssistant, uuid: "00000000-0000-4000-8000-000000000135" }),
13909+
);
13910+
assert.lengthOf(harness.continuationRequests, 2);
13911+
}).pipe(
13912+
Effect.provide(
13913+
Layer.mergeAll(
13914+
IdAllocator.layer,
13915+
McpProviderSessions.layer,
13916+
NodeServices.layer,
13917+
NodeCrypto.layer,
13918+
),
13919+
),
13920+
),
13921+
),
13922+
);
13923+
1376913924
it.effect.each([true, false])(
1377013925
"carries a rejected wake rate limit into the continuation failure (event=%s)",
1377113926
(standaloneEvent) =>

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5569,7 +5569,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
55695569
return { kind: "command", label, outcome };
55705570
}
55715571
});
5572-
const requestedContinuations = yield* Ref.make(new Set<string>());
5572+
const requestedContinuations = yield* Ref.make(new Map<string, symbol>());
55735573
const absorbedWakes = yield* Ref.make(new Set<string>());
55745574
const heldProposedPlansByToolUseId = new Map<string, string>();
55755575
const runtimeContext = yield* Effect.context<never>();
@@ -6052,7 +6052,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
60526052
if (!current.has(nativeThreadId)) {
60536053
return current;
60546054
}
6055-
const updated = new Set(current);
6055+
const updated = new Map(current);
60566056
updated.delete(nativeThreadId);
60576057
return updated;
60586058
});
@@ -8911,12 +8911,14 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
89118911
});
89128912
return;
89138913
}
8914+
const nativeThreadId = wakeInput.nativeThreadId;
8915+
const latch = Symbol(nativeThreadId);
89148916
const shouldOffer = yield* Ref.modify(requestedContinuations, (current) => {
8915-
if (current.has(wakeInput.nativeThreadId)) {
8917+
if (current.has(nativeThreadId)) {
89168918
return [false, current] as const;
89178919
}
8918-
const updated = new Set(current);
8919-
updated.add(wakeInput.nativeThreadId);
8920+
const updated = new Map(current);
8921+
updated.set(nativeThreadId, latch);
89208922
return [true, updated] as const;
89218923
});
89228924
if (!shouldOffer) {
@@ -8939,6 +8941,16 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
89398941
driver: CLAUDE_PROVIDER,
89408942
detail,
89418943
...(notification === null ? {} : { notification }),
8944+
// A process reset can clear and re-latch this thread; a stale failure must not release the newer wake.
8945+
clearIfCurrent: () =>
8946+
Ref.update(requestedContinuations, (current) => {
8947+
if (current.get(nativeThreadId) !== latch) {
8948+
return current;
8949+
}
8950+
const updated = new Map(current);
8951+
updated.delete(nativeThreadId);
8952+
return updated;
8953+
}),
89428954
});
89438955
});
89448956

@@ -11760,7 +11772,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
1176011772
yield* attachTurn;
1176111773
const drained = (yield* Ref.get(wakeBuffers)).get(nativeThreadId)?.messages ?? [];
1176211774
yield* Ref.update(requestedContinuations, (current) => {
11763-
const updated = new Set(current);
11775+
const updated = new Map(current);
1176411776
updated.delete(nativeThreadId);
1176511777
return updated;
1176611778
});

‎apps/server/src/orchestration-v2/ProviderContinuationService.test.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { assert, describe, it } from "@effect/vitest";
22
import {
3+
CommandId,
34
MessageId,
45
ProjectId,
56
ProviderDriverKind,
@@ -22,6 +23,7 @@ import * as TestClock from "effect/testing/TestClock";
2223

2324
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
2425
import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
26+
import * as Orchestrator from "./Orchestrator.ts";
2527
import * as ProviderContinuationService from "./ProviderContinuationService.ts";
2628
import * as ThreadManagementService from "./ThreadManagementService.ts";
2729

@@ -418,6 +420,43 @@ describe("ProviderContinuationService", () => {
418420
},
419421
);
420422

423+
it.effect(
424+
"clears a sticky continuation offer when a non-delegated wake dispatch fails generically",
425+
() => {
426+
return Effect.gen(function* () {
427+
const cleared = yield* Deferred.make<void>();
428+
const layerThreads = Layer.mock(ThreadManagementService.ThreadManagementService)({
429+
getThreadRecords: () => Effect.succeed(projection),
430+
dispatch: (command) =>
431+
Effect.fail(
432+
new Orchestrator.OrchestratorDispatchError({
433+
commandId: CommandId.make(String(command.commandId)),
434+
commandType: command.type,
435+
cause: new Error("SQLITE_FULL: database or disk is full"),
436+
}),
437+
),
438+
});
439+
const layerWorker = ProviderContinuationService.layer.pipe(
440+
Layer.provide(
441+
Layer.mergeAll(IdAllocator.layer, ProviderContinuationRequests.layer, layerThreads),
442+
),
443+
);
444+
445+
yield* Effect.gen(function* () {
446+
const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
447+
yield* requests.offer({
448+
...request(),
449+
clearIfCurrent: () => Deferred.succeed(cleared, undefined),
450+
});
451+
yield* Deferred.await(cleared);
452+
}).pipe(
453+
Effect.provide(Layer.merge(ProviderContinuationRequests.layer, layerWorker)),
454+
Effect.scoped,
455+
);
456+
});
457+
},
458+
);
459+
421460
it.effect(
422461
"drops stopped and disposed delegated completions instead of reviving them after recovery",
423462
() => {

‎apps/server/src/orchestration-v2/ProviderContinuationService.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,8 @@ export const layer = Layer.effectDiscard(
274274
),
275275
Effect.forkScoped,
276276
);
277+
} else if (request.clearIfCurrent !== undefined) {
278+
yield* request.clearIfCurrent();
277279
}
278280
}),
279281
),

0 commit comments

Comments
 (0)