Skip to content

Commit a9e9f71

Browse files
committed
fix(server): settle refused starts and reconcile session teardown
Settle unsupported compaction before claiming ownership and share the established pre-prompt refusal predicate. Preserve consumers after ambiguous acceptance errors. Revoke pending starts on confirmed exact-runtime teardown, await guarded failure finalization, and remove ownership before waking interrupt waiters. Add refusal and teardown regressions to the existing race table. Verification: check:changed against 337c585 passed all 36 package typechecks and 400 tests. External witnesses failed before the correction and passed afterward. Model: openai-codex/gpt-6.1-sol. Harness: Pi.
1 parent 7594589 commit a9e9f71

6 files changed

Lines changed: 279 additions & 56 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderSessionManager.ts‎

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,7 @@ interface LiveSessionEntry {
226226
readonly eventSubscribers: Ref.Ref<
227227
ReadonlyMap<number, Queue.Queue<ProviderSessionEventSignal, Cause.Done>>
228228
>;
229+
readonly closeObservers: Ref.Ref<ReadonlySet<Effect.Effect<void>> | null>;
229230
readonly requestEventPermit: Semaphore.Semaphore;
230231
readonly scope: Scope.Closeable;
231232
readonly idleGeneration: number;
@@ -971,9 +972,11 @@ export const layerWithOptions = (
971972
const closeScopeWithin = Effect.fn("ProviderSessionManager.closeScopeWithin")(function* (
972973
scope: Scope.Closeable,
973974
annotations: { readonly providerSessionId?: ProviderSessionId; readonly reason: string },
975+
onClosed: Effect.Effect<void> = Effect.void,
974976
) {
975977
const closeFiber = yield* Scope.close(scope, Exit.void).pipe(
976978
Effect.exit,
979+
Effect.tap((exit) => (Exit.isSuccess(exit) ? onClosed : Effect.void)),
977980
Effect.forkDetach({ startImmediately: true }),
978981
);
979982
const closeExit = yield* Fiber.join(closeFiber).pipe(
@@ -1039,10 +1042,18 @@ export const layerWithOptions = (
10391042
input.detail ?? `Provider session released: ${input.reason}.`,
10401043
);
10411044
}
1042-
const closeExit = yield* closeScopeWithin(entry.scope, {
1043-
providerSessionId: input.providerSessionId,
1044-
reason: input.reason,
1045-
});
1045+
const closeExit = yield* closeScopeWithin(
1046+
entry.scope,
1047+
{ providerSessionId: input.providerSessionId, reason: input.reason },
1048+
Ref.getAndSet(entry.closeObservers, null).pipe(
1049+
Effect.flatMap((observers) =>
1050+
Effect.forEach(observers ?? [], (observer) => observer, {
1051+
concurrency: "unbounded",
1052+
discard: true,
1053+
}),
1054+
),
1055+
),
1056+
);
10461057
const records = {
10471058
entry,
10481059
reason: input.reason,
@@ -1695,6 +1706,7 @@ export const layerWithOptions = (
16951706
eventSubscribers: Ref.Ref<
16961707
ReadonlyMap<number, Queue.Queue<ProviderSessionEventSignal, Cause.Done>>
16971708
>,
1709+
closeObservers: LiveSessionEntry["closeObservers"],
16981710
): ProviderAdapter.ProviderAdapterV2SessionRuntime => {
16991711
const providerSessionId = runtime.providerSessionId;
17001712
const subscribeEvents = makeEventSubscription(eventSubscribers);
@@ -1711,6 +1723,22 @@ export const layerWithOptions = (
17111723
return {
17121724
...runtime,
17131725
subscribeEvents,
1726+
onSessionClosed: (observer) =>
1727+
Ref.modify(closeObservers, (current) =>
1728+
current === null
1729+
? ([true, current] as const)
1730+
: ([false, new Set([...current, observer])] as const),
1731+
).pipe(
1732+
Effect.tap((closed) => (closed ? observer : Effect.void)),
1733+
Effect.as(
1734+
Ref.update(closeObservers, (current) => {
1735+
if (current === null) return null;
1736+
const next = new Set(current);
1737+
next.delete(observer);
1738+
return next;
1739+
}),
1740+
),
1741+
),
17141742
events: Stream.unwrap(
17151743
subscribeEvents.pipe(Effect.map((subscription) => subscription.events)),
17161744
),
@@ -2232,7 +2260,10 @@ export const layerWithOptions = (
22322260
const eventSubscribers = yield* Ref.make<
22332261
ReadonlyMap<number, Queue.Queue<ProviderSessionEventSignal, Cause.Done>>
22342262
>(new Map());
2235-
const exposedRuntime = decorateRuntime(runtime, eventSubscribers);
2263+
const closeObservers = yield* Ref.make<ReadonlySet<Effect.Effect<void>> | null>(
2264+
new Set(),
2265+
);
2266+
const exposedRuntime = decorateRuntime(runtime, eventSubscribers, closeObservers);
22362267
const entry: LiveSessionEntry = {
22372268
attachedThreadIds: new Set([input.threadId]),
22382269
loadedProviderThreadKeyByThread: new Map(),
@@ -2246,6 +2277,7 @@ export const layerWithOptions = (
22462277
runtime,
22472278
exposedRuntime,
22482279
eventSubscribers,
2280+
closeObservers,
22492281
requestEventPermit: yield* Semaphore.make(1),
22502282
scope: sessionScope,
22512283
idleGeneration: 0,

‎apps/server/src/orchestration-v2/ProviderStartStop.integration.test.ts‎

Lines changed: 128 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import {
1212
type OrchestrationV2ProviderTurn,
1313
} from "@t3tools/contracts";
1414
import * as DateTime from "effect/DateTime";
15+
import * as Cause from "effect/Cause";
1516
import * as Config from "effect/Config";
1617
import * as Logger from "effect/Logger";
1718
import * as References from "effect/References";
@@ -33,6 +34,7 @@ import * as EventSink from "./EventSink.ts";
3334
import * as Orchestrator from "./Orchestrator.ts";
3435
import * as ProjectionStore from "./ProjectionStore.ts";
3536
import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts";
37+
import * as ProviderSessionManager from "./ProviderSessionManager.ts";
3638
import * as RunExecutionService from "./RunExecutionService.ts";
3739
import * as ThreadManagementService from "./ThreadManagementService.ts";
3840
import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts";
@@ -49,11 +51,17 @@ it.effect.each([
4951
"interrupt-timeout",
5052
"identity-timeout",
5153
"identity-timeout-force",
54+
"unsupported-compact",
55+
"prompt-refusal",
56+
"manager-release",
57+
"manager-release-interrupt",
58+
"provider-stopped",
59+
"stream-ended",
5260
] as const)("Stop crosses native start: %s", (scenario) =>
5361
Effect.scoped(
5462
Effect.gen(function* () {
5563
const cwd = yield* checkpointWorkspace(`start-stop-${scenario}`);
56-
const events = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
64+
const events = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event, Cause.Done>();
5765
const entered = yield* Deferred.make<void>();
5866
const accept = yield* Deferred.make<void>();
5967
const nativeAccepted = yield* Deferred.make<void>();
@@ -94,6 +102,18 @@ it.effect.each([
94102
turn,
95103
) =>
96104
Effect.gen(function* () {
105+
if (scenario === "prompt-refusal") {
106+
return yield* new ProviderAdapter.ProviderAdapterTurnStartError({
107+
driver,
108+
threadId: turn.threadId,
109+
providerThreadId: turn.providerThread.id,
110+
runId: turn.runId,
111+
cause: {
112+
_tag: "ClaudeAppendSystemPromptFileError",
113+
path: "/task-owned/missing-file",
114+
},
115+
});
116+
}
97117
yield* Deferred.succeed(entered, undefined);
98118
yield* Deferred.await(accept);
99119
assert.isUndefined(active, "native turns must never overlap");
@@ -290,6 +310,15 @@ it.effect.each([
290310
const originalStart = execution.startRootRun;
291311
let staleInput: RunExecutionService.RunExecutionServiceV2StartRootRunInput | undefined;
292312
using _beforeOwnership = vi.spyOn(execution, "startRootRun").mockImplementation((input) => {
313+
if (scenario === "unsupported-compact") {
314+
const session = { ...input.session };
315+
delete session.compactThread;
316+
return originalStart({
317+
...input,
318+
message: { ...input.message, text: "/compact" },
319+
session,
320+
});
321+
}
293322
if (scenario !== "before-ownership") return originalStart(input);
294323
staleInput = input;
295324
return Deferred.succeed(beforeOwnership, undefined).pipe(Effect.andThen(Effect.never));
@@ -312,6 +341,26 @@ it.effect.each([
312341
});
313342
yield* send("first");
314343
const startup = yield* worker.drain(1).pipe(Effect.forkScoped);
344+
if (scenario === "unsupported-compact" || scenario === "prompt-refusal") {
345+
yield* Fiber.join(startup);
346+
const failed = yield* orchestrator.getThreadProjection(threadId);
347+
const run = failed.runs[0]!;
348+
assert.equal(run.status, "failed");
349+
assert.equal(failed.attempts[0]?.status, "failed");
350+
assert.equal(failed.nodes.find((node) => node.id === run.rootNodeId)?.status, "failed");
351+
assert.isTrue(
352+
Option.isNone(
353+
yield* execution.getStartOwnership({
354+
threadId,
355+
runId: run.id,
356+
attemptId: run.activeAttemptId!,
357+
}),
358+
),
359+
);
360+
assert.equal(starts, 0);
361+
assert.deepEqual(interrupts, []);
362+
return;
363+
}
315364
if (scenario === "before-ownership") {
316365
yield* Deferred.await(beforeOwnership);
317366
yield* orchestrator.dispatch({
@@ -333,6 +382,75 @@ it.effect.each([
333382
if (scenario === "queued-promotion") yield* send("promoted", true);
334383
const initial = yield* orchestrator.getThreadProjection(threadId);
335384
const run = initial.runs.find((run) => run.status === "running")!;
385+
if (
386+
[
387+
"manager-release",
388+
"manager-release-interrupt",
389+
"provider-stopped",
390+
"stream-ended",
391+
].includes(scenario)
392+
) {
393+
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
394+
const providerSessionId = initial.providerThreads.find(
395+
(thread) => thread.id === run.providerThreadId,
396+
)!.providerSessionId!;
397+
const runtime = Option.getOrThrow(yield* manager.get(providerSessionId));
398+
if (scenario === "manager-release-interrupt") {
399+
const accepted = yield* orchestrator.streamDomainEvents.pipe(
400+
Stream.filter(
401+
(event) =>
402+
event.type === "provider-turn.updated" && event.payload.status === "running",
403+
),
404+
Stream.take(1),
405+
Stream.runDrain,
406+
Effect.forkScoped,
407+
);
408+
yield* Deferred.succeed(accept, undefined);
409+
yield* Fiber.join(accepted);
410+
}
411+
const waiter = yield* Deferred.succeed(ownedWait, undefined).pipe(
412+
Effect.andThen(
413+
execution.interruptOwnedStart({
414+
threadId,
415+
runId: run.id,
416+
attemptId: run.activeAttemptId!,
417+
}),
418+
),
419+
Effect.forkScoped,
420+
);
421+
yield* Deferred.await(ownedWait);
422+
if (scenario === "manager-release-interrupt") yield* Deferred.await(interruptCalled);
423+
if (scenario === "provider-stopped" || scenario === "stream-ended") {
424+
if (scenario === "provider-stopped")
425+
yield* Queue.offer(events, {
426+
type: "provider_session.updated",
427+
driver,
428+
providerSession: { ...runtime.providerSession, status: "stopped" },
429+
});
430+
yield* Queue.end(events);
431+
} else {
432+
yield* manager.release({ providerSessionId, reason: "manual_shutdown" });
433+
}
434+
yield* Fiber.join(waiter);
435+
yield* Fiber.join(startup);
436+
assert.isTrue(
437+
Option.isNone(
438+
yield* execution.getStartOwnership({
439+
threadId,
440+
runId: run.id,
441+
attemptId: run.activeAttemptId!,
442+
}),
443+
),
444+
);
445+
const failed = yield* orchestrator.getThreadProjection(threadId);
446+
assert.equal(failed.runs[0]?.status, "failed");
447+
assert.equal(failed.attempts[0]?.status, "failed");
448+
assert.isFalse(failed.turnItems.some((item) => item.type === "run_interrupt_result"));
449+
yield* Deferred.succeed(accept, undefined);
450+
yield* Deferred.succeed(releaseStart, undefined);
451+
assert.equal(starts, scenario === "manager-release-interrupt" ? 1 : 0);
452+
return;
453+
}
336454
const original = store.getThreadRecords;
337455
let armed = true;
338456
using _snapshot = vi.spyOn(store, "getThreadRecords").mockImplementation((...args) =>
@@ -448,6 +566,15 @@ it.effect.each([
448566
assert.deepEqual(interrupts, []);
449567
const outcome = yield* threads.forceStopThread({ threadId, runId: run.id });
450568
assert.equal(outcome.outcome, "released");
569+
assert.isTrue(
570+
Option.isNone(
571+
yield* execution.getStartOwnership({
572+
threadId,
573+
runId: run.id,
574+
attemptId: run.activeAttemptId!,
575+
}),
576+
),
577+
);
451578
const forced = yield* orchestrator.getThreadProjection(threadId);
452579
assert.equal(forced.runs[0]?.status, "interrupted");
453580
assert.isTrue(

‎apps/server/src/orchestration-v2/ProviderTurnStartService.ts‎

Lines changed: 1 addition & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ import * as FileSystem from "effect/FileSystem";
2222
import * as Layer from "effect/Layer";
2323
import * as Option from "effect/Option";
2424
import * as Path from "effect/Path";
25-
import * as Predicate from "effect/Predicate";
2625
import * as Schema from "effect/Schema";
2726

2827
import * as ServerConfig from "../config.ts";
@@ -54,6 +53,7 @@ import {
5453
ContextHandoffBackgroundWorkError,
5554
knownProviderFailureMessage,
5655
makeProviderFailure,
56+
refusedBeforePrompt,
5757
} from "@t3tools/provider-core/server/failure";
5858
import * as RunExecutionService from "./RunExecutionService.ts";
5959
import * as RuntimePolicy from "./RuntimePolicy.ts";
@@ -98,23 +98,6 @@ const isProviderAdapterOpenSessionError = Schema.is(
9898
ProviderAdapter.ProviderAdapterOpenSessionError,
9999
);
100100

101-
/**
102-
* Claude refuses these before it reads the prompt: replacing a process running background work,
103-
* launch arguments whose appended system prompt or settings cannot be read, or MCP registration
104-
* that never answers while the query opens.
105-
*/
106-
const PROMPT_REFUSAL_TAGS = [
107-
"ClaudeBackgroundWorkBlocksQueryReplacementError",
108-
"ClaudeAppendSystemPromptFileError",
109-
"ClaudeAppendSystemPromptMissingValueError",
110-
"ClaudeSettingsLaunchArgError",
111-
"ClaudeMcpRegistrationTimeoutError",
112-
] as const;
113-
114-
const refusedBeforePrompt = (error: unknown): boolean =>
115-
PROMPT_REFUSAL_TAGS.some((tag) => Predicate.isTagged(error, tag)) ||
116-
(Predicate.hasProperty(error, "cause") && refusedBeforePrompt(error.cause));
117-
118101
export interface ProviderTurnStartServiceV2Shape {
119102
readonly start: (input: {
120103
readonly threadId: ThreadId;

0 commit comments

Comments
 (0)