Skip to content

Commit f8fe927

Browse files
committed
fix(server): stop superseded attempts before native setup
1 parent efecd3c commit f8fe927

2 files changed

Lines changed: 224 additions & 39 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts‎

Lines changed: 222 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { expect, it, vi } from "vite-plus/test";
22
import { it as effectIt } from "@effect/vitest";
33
import {
44
CheckpointScopeId,
5+
ContextTransferId,
56
MessageId,
67
NodeId,
78
ProviderSessionId,
@@ -18,7 +19,9 @@ import {
1819
} from "@t3tools/contracts";
1920
import * as Cause from "effect/Cause";
2021
import * as DateTime from "effect/DateTime";
22+
import * as Deferred from "effect/Deferred";
2123
import * as Effect from "effect/Effect";
24+
import * as Fiber from "effect/Fiber";
2225
import * as FileSystem from "effect/FileSystem";
2326
import * as Layer from "effect/Layer";
2427
import * as Option from "effect/Option";
@@ -168,6 +171,11 @@ function makeLocalCommandHarness(input: {
168171
readonly historyReadFailureAfterFallback?: unknown;
169172
readonly interruptOpen?: boolean;
170173
readonly interruptRunBeforeOpenFailure?: boolean;
174+
readonly interruptRunBeforeOpen?: boolean;
175+
readonly interruptRunDuringOpen?: boolean;
176+
readonly policyGate?: Effect.Effect<void>;
177+
readonly openGate?: Effect.Effect<void>;
178+
readonly nativeLoadMode?: "ensure" | "resume" | "fork";
171179
readonly writeFailure?: unknown;
172180
/** Loads the thread and starts the run, then fails every later state read. */
173181
readonly failReadsAfterRunning?: boolean;
@@ -355,6 +363,49 @@ function makeLocalCommandHarness(input: {
355363
),
356364
};
357365
}
366+
if (input.nativeLoadMode === "resume") {
367+
projection = {
368+
...projection,
369+
providerThreads: projection.providerThreads.map((candidate) =>
370+
candidate.id === providerThreadId
371+
? {
372+
...candidate,
373+
nativeThreadRef: {
374+
driver: candidate.driver,
375+
nativeId: "existing-native-thread",
376+
strength: "strong",
377+
},
378+
}
379+
: candidate,
380+
),
381+
};
382+
}
383+
if (input.nativeLoadMode === "fork") {
384+
const sourceThreadId = ThreadId.make("native-fork-source");
385+
projection = {
386+
...projection,
387+
contextTransfers: [
388+
{
389+
id: ContextTransferId.make("pending-native-fork"),
390+
type: "fork",
391+
sourceThreadId,
392+
targetThreadId: threadId,
393+
sourcePoint: { threadId: sourceThreadId, runId: RunId.make("source-run") },
394+
basePoint: null,
395+
sourceProviderInstanceId: newInstanceId,
396+
targetProviderInstanceId: newInstanceId,
397+
targetRunId: runId,
398+
status: "pending",
399+
resolution: null,
400+
createdBy: "user",
401+
error: null,
402+
createdAt: now,
403+
updatedAt: now,
404+
consumedAt: null,
405+
},
406+
],
407+
};
408+
}
358409
const events: Array<OrchestrationV2DomainEvent> = [];
359410
const interruptRun = () => {
360411
projection = {
@@ -393,45 +444,61 @@ function makeLocalCommandHarness(input: {
393444
),
394445
ensureThread: () => Effect.succeed(providerThread),
395446
};
447+
const resumeThread = vi.fn(() => Effect.die("A superseded attempt must not resume a thread."));
448+
const forkThread = vi.fn(() => Effect.die("A superseded attempt must not fork a thread."));
396449
const open = vi.fn(() =>
397-
input.interruptOpen === true
398-
? Effect.interrupt
399-
: "historyReadFailureAfterFallback" in input
400-
? Effect.succeed(resumeFallbackSession as never)
401-
: "ensureThreadFailure" in input
402-
? Effect.succeed({ driver: providerThread.driver, ensureThread } as never)
403-
: "openFailure" in input
404-
? Effect.sync(() => {
405-
if (input.interruptRunBeforeOpenFailure === true) interruptRun();
406-
}).pipe(
407-
Effect.andThen(
408-
Effect.fail(
409-
new ProviderSessionManager.ProviderSessionOpenError({
410-
instanceId: newInstanceId,
411-
providerSessionId,
412-
cause: input.openFailure,
413-
}),
414-
),
415-
),
416-
)
417-
: input.failReadsAfterRunning === true
418-
? Effect.succeed({
419-
driver: providerThread.driver,
420-
providerSession: {
421-
id: providerSessionId,
422-
driver: providerThread.driver,
423-
providerInstanceId: newInstanceId,
424-
status: "ready",
425-
cwd: "/tmp/native-account-command",
426-
model: null,
427-
capabilities: CodexProviderCapabilitiesV2,
428-
createdAt: now,
429-
updatedAt: now,
430-
lastError: null,
431-
},
432-
ensureThread: () => Effect.succeed(providerThread),
433-
} as never)
434-
: Effect.die("A local command must not open a native session."),
450+
input.openGate !== undefined
451+
? input.openGate.pipe(
452+
Effect.as({
453+
driver: providerThread.driver,
454+
ensureThread,
455+
resumeThread,
456+
forkThread,
457+
} as never),
458+
)
459+
: input.interruptOpen === true
460+
? Effect.interrupt
461+
: input.interruptRunDuringOpen === true
462+
? Effect.sync(() => {
463+
interruptRun();
464+
return { driver: providerThread.driver, ensureThread } as never;
465+
})
466+
: "historyReadFailureAfterFallback" in input
467+
? Effect.succeed(resumeFallbackSession as never)
468+
: "ensureThreadFailure" in input
469+
? Effect.succeed({ driver: providerThread.driver, ensureThread } as never)
470+
: "openFailure" in input
471+
? Effect.sync(() => {
472+
if (input.interruptRunBeforeOpenFailure === true) interruptRun();
473+
}).pipe(
474+
Effect.andThen(
475+
Effect.fail(
476+
new ProviderSessionManager.ProviderSessionOpenError({
477+
instanceId: newInstanceId,
478+
providerSessionId,
479+
cause: input.openFailure,
480+
}),
481+
),
482+
),
483+
)
484+
: input.failReadsAfterRunning === true
485+
? Effect.succeed({
486+
driver: providerThread.driver,
487+
providerSession: {
488+
id: providerSessionId,
489+
driver: providerThread.driver,
490+
providerInstanceId: newInstanceId,
491+
status: "ready",
492+
cwd: "/tmp/native-account-command",
493+
model: null,
494+
capabilities: CodexProviderCapabilitiesV2,
495+
createdAt: now,
496+
updatedAt: now,
497+
lastError: null,
498+
},
499+
ensureThread: () => Effect.succeed(providerThread),
500+
} as never)
501+
: Effect.die("A local command must not open a native session."),
435502
);
436503
const startRootRun = vi.fn<
437504
(input: RunExecutionService.RunExecutionServiceV2StartRootRunInput) => Effect.Effect<void>
@@ -495,6 +562,14 @@ function makeLocalCommandHarness(input: {
495562
Layer.mock(GitWorkflow.GitWorkflowService)({}),
496563
Layer.mock(ProjectService.ProjectService)({}),
497564
Layer.mock(ProjectionStore.ProjectionStoreV2)({
565+
getThreadRecords: () =>
566+
Effect.succeed({
567+
...projection,
568+
runs: projection.runs.map((candidate) => ({
569+
...candidate,
570+
id: RunId.make("source-run"),
571+
})),
572+
}),
498573
getTurnStartContext: () =>
499574
Effect.succeed({
500575
...projection,
@@ -525,20 +600,48 @@ function makeLocalCommandHarness(input: {
525600
Layer.mock(ProviderAuthService.ProviderAuthService)({ tryHandlePromptCommand }),
526601
Layer.mock(RunExecutionService.RunExecutionServiceV2)({ startRootRun }),
527602
Layer.mock(RuntimePolicy.RuntimePolicyV2)({
528-
resolve: () => Effect.succeed({} as never),
603+
resolve: () =>
604+
(input.policyGate ?? Effect.void).pipe(
605+
Effect.andThen(
606+
Effect.sync(() => {
607+
if (input.interruptRunBeforeOpen === true) interruptRun();
608+
return {} as never;
609+
}),
610+
),
611+
),
529612
}),
530613
),
531614
),
532615
);
533616
return {
534617
open,
618+
ensureThread,
619+
resumeThread,
620+
forkThread,
535621
writeIfRunCurrent,
536622
startRootRun,
537623
tryHandlePromptCommand,
538624
events,
539625
oldInstanceId,
540626
newInstanceId,
541627
attemptId,
628+
supersedeAttempt: () => {
629+
const replacementId = RunAttemptId.make("replacement-starting-attempt");
630+
projection = {
631+
...projection,
632+
runs: projection.runs.map((candidate) =>
633+
candidate.id === runId ? { ...candidate, activeAttemptId: replacementId } : candidate,
634+
),
635+
attempts: [
636+
...projection.attempts.map((candidate) =>
637+
candidate.id === attemptId
638+
? { ...candidate, status: "interrupted" as const, completedAt: now }
639+
: candidate,
640+
),
641+
{ ...projection.attempts[0]!, id: replacementId, attemptOrdinal: 2 },
642+
],
643+
};
644+
},
542645
projection: () => projection,
543646
start: Effect.gen(function* () {
544647
yield* (yield* ProviderTurnStart.ProviderTurnStartServiceV2).start({ threadId, runId });
@@ -553,6 +656,85 @@ function makeLocalCommandHarness(input: {
553656
};
554657
}
555658

659+
effectIt.effect(
660+
"does not open a session for an attempt replaced during suspended policy resolution",
661+
() =>
662+
Effect.gen(function* () {
663+
const entered = yield* Deferred.make<void>();
664+
const release = yield* Deferred.make<void>();
665+
const harness = makeLocalCommandHarness({
666+
text: "Continue",
667+
policyGate: Deferred.succeed(entered, undefined).pipe(
668+
Effect.andThen(Deferred.await(release)),
669+
),
670+
});
671+
const startup = yield* harness.start.pipe(Effect.forkScoped);
672+
yield* Deferred.await(entered);
673+
harness.supersedeAttempt();
674+
yield* Deferred.succeed(release, undefined);
675+
yield* Fiber.join(startup);
676+
expect(harness.projection().runs.at(-1)?.status).toBe("starting");
677+
expect(harness.projection().runs.at(-1)?.activeAttemptId).not.toBe(harness.attemptId);
678+
expect(harness.open).not.toHaveBeenCalled();
679+
expect(harness.writeIfRunCurrent).not.toHaveBeenCalled();
680+
expect(harness.startRootRun).not.toHaveBeenCalled();
681+
}).pipe(Effect.scoped),
682+
);
683+
684+
effectIt.effect.each(["ensure", "resume", "fork"] as const)(
685+
"does not %s a native thread for an attempt replaced during suspended session opening",
686+
(nativeLoadMode) =>
687+
Effect.gen(function* () {
688+
const entered = yield* Deferred.make<void>();
689+
const release = yield* Deferred.make<void>();
690+
const harness = makeLocalCommandHarness({
691+
text: "Continue",
692+
nativeLoadMode,
693+
openGate: Deferred.succeed(entered, undefined).pipe(
694+
Effect.andThen(Deferred.await(release)),
695+
),
696+
});
697+
const startup = yield* harness.start.pipe(Effect.forkScoped);
698+
yield* Deferred.await(entered);
699+
harness.supersedeAttempt();
700+
yield* Deferred.succeed(release, undefined);
701+
yield* Fiber.join(startup);
702+
expect(harness.projection().runs.at(-1)?.status).toBe("starting");
703+
expect(harness.projection().runs.at(-1)?.activeAttemptId).not.toBe(harness.attemptId);
704+
expect(harness.open).toHaveBeenCalledOnce();
705+
expect(harness.ensureThread).not.toHaveBeenCalled();
706+
expect(harness.resumeThread).not.toHaveBeenCalled();
707+
expect(harness.forkThread).not.toHaveBeenCalled();
708+
expect(harness.writeIfRunCurrent).not.toHaveBeenCalled();
709+
expect(harness.startRootRun).not.toHaveBeenCalled();
710+
expect(harness.events).toHaveLength(0);
711+
}).pipe(Effect.scoped),
712+
);
713+
714+
effectIt.effect("does not open a session after the attempt stops during policy resolution", () =>
715+
Effect.gen(function* () {
716+
const harness = makeLocalCommandHarness({ text: "Continue", interruptRunBeforeOpen: true });
717+
yield* harness.start;
718+
expect(harness.open).not.toHaveBeenCalled();
719+
expect(harness.startRootRun).not.toHaveBeenCalled();
720+
expect(harness.projection().runs.at(-1)?.status).toBe("interrupted");
721+
}),
722+
);
723+
724+
effectIt.effect(
725+
"does not load a provider thread after the attempt stops while opening its session",
726+
() =>
727+
Effect.gen(function* () {
728+
const harness = makeLocalCommandHarness({ text: "Continue", interruptRunDuringOpen: true });
729+
yield* harness.start;
730+
expect(harness.open).toHaveBeenCalledOnce();
731+
expect(harness.ensureThread).not.toHaveBeenCalled();
732+
expect(harness.startRootRun).not.toHaveBeenCalled();
733+
expect(harness.events).toHaveLength(0);
734+
expect(harness.projection().runs.at(-1)?.status).toBe("interrupted");
735+
}),
736+
);
737+
556738
effectIt.effect("terminalizes a starting run when its provider session cannot open", () =>
557739
Effect.gen(function* () {
558740
const harness = makeLocalCommandHarness({
@@ -563,6 +745,7 @@ effectIt.effect("terminalizes a starting run when its provider session cannot op
563745
yield* harness.start;
564746

565747
expect(harness.open).toHaveBeenCalledOnce();
748+
expect(harness.ensureThread).not.toHaveBeenCalled();
566749
expect(harness.startRootRun).not.toHaveBeenCalled();
567750
expect(harness.writeIfRunCurrent).toHaveBeenCalledWith(
568751
expect.objectContaining({

‎apps/server/src/orchestration-v2/ProviderTurnStartService.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -519,6 +519,7 @@ export const layer: Layer.Layer<
519519
thread: projection.thread,
520520
modelSelection: run.modelSelection,
521521
});
522+
if (!(yield* isCurrentAttemptInStatus("starting"))) return;
522523
const existingSessionProjection = projection.providerSessions.find(
523524
(candidate) => candidate.id === providerSessionId,
524525
);
@@ -585,6 +586,7 @@ export const layer: Layer.Layer<
585586
return;
586587
}
587588
const session = sessionResult.success;
589+
if (!(yield* isCurrentAttemptInStatus("starting"))) return;
588590
// Only the provider's own thread load fails the run on the last attempt;
589591
// store, id and handoff failures around it keep their typed errors.
590592
const loadFromProvider = (

0 commit comments

Comments
 (0)