Skip to content

Commit 972a3d0

Browse files
committed
fix(server): stop retained work after target startup failure
1 parent fc26dd3 commit 972a3d0

2 files changed

Lines changed: 98 additions & 13 deletions

File tree

‎apps/server/src/orchestration-v2/ClaudeAccountSwitchWake.integration.test.ts‎

Lines changed: 84 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,15 @@ import * as SecureRandom from "effect/crypto/SecureRandom";
1616
import * as Effect from "effect/Effect";
1717
import * as FileSystem from "effect/FileSystem";
1818
import * as Fiber from "effect/Fiber";
19+
import * as Exit from "effect/Exit";
1920
import * as Layer from "effect/Layer";
21+
import * as Option from "effect/Option";
2022
import * as Path from "effect/Path";
2123
import * as Queue from "effect/Queue";
2224
import * as Schema from "effect/Schema";
2325
import * as Stream from "effect/Stream";
2426
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
27+
import { ProviderAdapterOpenSessionError } from "@t3tools/provider-core/server/ProviderAdapter";
2528
import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
2629
import type * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
2730
import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace";
@@ -32,6 +35,7 @@ import * as EffectWorker from "./EffectWorker.ts";
3235
import * as Orchestrator from "./Orchestrator.ts";
3336
import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts";
3437
import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts";
38+
import * as ProviderSessionManager from "./ProviderSessionManager.ts";
3539

3640
const nativeSessionId = "account-switch-wake-session";
3741
const settings = Schema.decodeSync(ClaudeSettings)({});
@@ -165,8 +169,13 @@ const makeAccount = Effect.fn("makeAccount")(function* (input: {
165169
return { adapter, processes };
166170
});
167171

168-
it.live.each(["sent after", "queued behind", "stopped after direct handoff"] as const)(
169-
"preserves Claude background-work ownership for a switch %s",
172+
it.live.each([
173+
"sent after",
174+
"queued behind",
175+
"stopped after direct handoff",
176+
"stopped after failed target start",
177+
] as const)(
178+
"preserves Claude background-work ownership and target selection for a switch %s",
170179
(variant) =>
171180
Effect.scoped(
172181
Effect.gen(function* () {
@@ -186,7 +195,25 @@ it.live.each(["sent after", "queued behind", "stopped after direct handoff"] as
186195
continuations,
187196
closed,
188197
});
189-
const adapters = [first.adapter, second.adapter];
198+
const failedStart = variant === "stopped after failed target start";
199+
const stopping = variant.startsWith("stopped after");
200+
const adapters = [
201+
first.adapter,
202+
failedStart
203+
? {
204+
...second.adapter,
205+
openSession: (input: Parameters<typeof second.adapter.openSession>[0]) =>
206+
Effect.fail(
207+
new ProviderAdapterOpenSessionError({
208+
driver: ClaudeAdapterV2.CLAUDE_PROVIDER,
209+
providerSessionId: input.providerSessionId,
210+
requiresSignIn: true,
211+
cause: "Controlled target sign-in refusal before first provider turn",
212+
}),
213+
),
214+
}
215+
: second.adapter,
216+
];
190217
// Two accounts of one Claude install resume each other's native history.
191218
const registry = Layer.succeed(
192219
ProviderAdapterRegistry.ProviderAdapterRegistryV2,
@@ -203,10 +230,9 @@ it.live.each(["sent after", "queued behind", "stopped after direct handoff"] as
203230
getMetadata: (instanceId) =>
204231
Effect.succeed({
205232
driver: ClaudeAdapterV2.CLAUDE_PROVIDER,
206-
continuationKey:
207-
variant === "stopped after direct handoff"
208-
? `claude:${instanceId}`
209-
: "claude:shared-native-account-history",
233+
continuationKey: stopping
234+
? `claude:${instanceId}`
235+
: "claude:shared-native-account-history",
210236
enabled: true,
211237
capabilities: ClaudeProviderCapabilitiesV2,
212238
}),
@@ -275,18 +301,20 @@ it.live.each(["sent after", "queued behind", "stopped after direct handoff"] as
275301

276302
if (!queued) yield* send(2, secondAccount);
277303
yield* worker.drain();
278-
if (variant === "stopped after direct handoff") {
279-
const secondSettling = yield* runStatus("send-2", "waiting");
280-
yield* Queue.offer(second.processes[0]!, resultFrame("turn-2", "Handoff complete."));
281-
yield* Fiber.join(secondSettling);
282-
yield* worker.drain();
304+
if (stopping) {
305+
if (!failedStart) {
306+
const secondSettling = yield* runStatus("send-2", "waiting");
307+
yield* Queue.offer(second.processes[0]!, resultFrame("turn-2", "Handoff complete."));
308+
yield* Fiber.join(secondSettling);
309+
yield* worker.drain();
310+
}
283311
const beforeStop = yield* orchestrator.getThreadProjection(threadId);
284312
const source = beforeStop.providerThreads.find(
285313
(thread) => thread.providerInstanceId === firstAccount.instanceId,
286314
)!;
287315
assert.deepEqual(
288316
beforeStop.runs.map((run) => run.status),
289-
["completed", "completed"],
317+
["completed", failedStart ? "failed" : "completed"],
290318
);
291319
assert.equal(source.pendingBackgroundTasks?.[0]?.taskId, "bash-build");
292320
assert.isFalse(
@@ -296,6 +324,32 @@ it.live.each(["sent after", "queued behind", "stopped after direct handoff"] as
296324
"the background command is reported only by the roster",
297325
);
298326
assert.deepEqual(closed, []);
327+
const sourceSession = Option.getOrThrow(
328+
yield* (yield* ProviderSessionManager.ProviderSessionManagerV2).get(
329+
source.providerSessionId!,
330+
),
331+
);
332+
assert.isTrue(yield* sourceSession.hasPendingBackgroundWorkForThread!(source));
333+
const target = beforeStop.runs[1]!;
334+
if (failedStart) {
335+
assert.isFalse(
336+
beforeStop.providerTurns.some(
337+
(turn) => turn.providerThreadId === target.providerThreadId,
338+
),
339+
);
340+
const directInterrupt = yield* orchestrator
341+
.dispatch({
342+
type: "run.interrupt",
343+
commandId: CommandId.make("direct-target-stop"),
344+
threadId,
345+
runId: target.id,
346+
})
347+
.pipe(Effect.exit);
348+
assert.isTrue(
349+
Exit.isFailure(directInterrupt),
350+
"a failed target alone is not interruptible",
351+
);
352+
}
299353
yield* orchestrator.dispatch({
300354
type: "thread.stop",
301355
commandId: CommandId.make("stop"),
@@ -313,6 +367,23 @@ it.live.each(["sent after", "queued behind", "stopped after direct handoff"] as
313367
?.pendingBackgroundTasks,
314368
[],
315369
);
370+
assert.isFalse(yield* sourceSession.hasPendingBackgroundWorkForThread!(source));
371+
if (failedStart) {
372+
assert.deepEqual(stopped.runs[1], target, "Stop preserves the target failure");
373+
assert.deepEqual(stopped.attempts, beforeStop.attempts);
374+
assert.deepEqual(stopped.nodes, beforeStop.nodes);
375+
assert.deepEqual(stopped.providerTurns, beforeStop.providerTurns);
376+
assert.deepEqual(
377+
stopped.turnItems.filter((item) => item.type === "error"),
378+
beforeStop.turnItems.filter((item) => item.type === "error"),
379+
);
380+
assert.isTrue(
381+
stopped.turnItems.some(
382+
(item) => item.runId === target.id && item.type === "run_interrupt_request",
383+
),
384+
"Stop still guards late tool calls on the failed target",
385+
);
386+
}
316387
assert.equal(
317388
stopped.thread.activeProviderThreadId,
318389
beforeStop.thread.activeProviderThreadId,

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13018,6 +13018,20 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1301813018
}
1301913019

1302013020
if (providerTurn === undefined) {
13021+
if (command.holdQueue === true && otherProviderInterrupts.length > 0) {
13022+
yield* emitEvent({
13023+
type: "turn-item.updated",
13024+
threadId: command.threadId,
13025+
runId: run.id,
13026+
nodeId: rootNode.id,
13027+
providerInstanceId: run.providerInstanceId,
13028+
occurredAt: now,
13029+
payload: interruptRequestItem,
13030+
});
13031+
yield* stopRemainingWork();
13032+
yield* Ref.update(effects, (existing) => [...existing, ...otherProviderInterrupts]);
13033+
return undefined;
13034+
}
1302113035
return yield* new OrchestratorDispatchError({
1302213036
commandId: command.commandId,
1302313037
commandType: command.type,

0 commit comments

Comments
 (0)