Skip to content

Commit b001943

Browse files
lidge-junCodex
andauthored
fix(server): complete ordinary-thread fallback triggers (#698)
* fix(server): complete ordinary-thread fallback triggers * fix(server): preserve recovery policy and guard fallback selection * fix(server): keep Fallback.make exported for the runtime layer test --------- Co-authored-by: Codex <codex@localhost>
1 parent 912de8a commit b001943

8 files changed

Lines changed: 958 additions & 66 deletions

File tree

‎apps/server/src/fleet/resources/Fallback.test.ts‎

Lines changed: 105 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ const shell = (overrides: Partial<OrchestrationV2ThreadShell> = {}) =>
5353

5454
const run = (ordinal: number, status: string, modelSelection: typeof opus) => ({
5555
id: RunId.make("run:" + ordinal),
56+
rootNodeId: null,
5657
ordinal,
5758
status,
5859
startedAt: ordinal,
@@ -61,6 +62,7 @@ const run = (ordinal: number, status: string, modelSelection: typeof opus) => ({
6162
});
6263

6364
interface World {
65+
stopped: boolean;
6466
shell: OrchestrationV2ThreadShell;
6567
runs: ReadonlyArray<ReturnType<typeof run>>;
6668
refuse: ReadonlySet<string>;
@@ -70,6 +72,7 @@ interface World {
7072
readonly commands: Array<OrchestrationV2ServerCommand>;
7173
}
7274
const makeWorld = (overrides: Partial<World> = {}): World => ({
75+
stopped: false,
7376
shell: shell(),
7477
runs: [run(1, "completed", opus), run(2, "failed", opus)],
7578
refuse: new Set(),
@@ -110,13 +113,39 @@ const threadChain: FleetSettings = {
110113
const threadsLayer = (world: World) =>
111114
Layer.mock(ThreadManagement.ThreadManagementService)({
112115
getThreadShell: () => Effect.sync(() => world.shell),
113-
getThreadRecords: (() => Effect.sync(() => ({ runs: world.runs }))) as never,
116+
getThreadRecords: (() =>
117+
Effect.sync(() => ({
118+
runs: world.runs,
119+
providerSessions: [],
120+
turnItems: [
121+
...(world.stopped ? [{ runId, type: "run_interrupt_request" }] : []),
122+
{
123+
id: "failure",
124+
runId,
125+
nodeId: null,
126+
type: "error",
127+
status: "failed",
128+
ordinal: 1,
129+
updatedAt: DateTime.makeUnsafe(FAILED_AT),
130+
failure: {
131+
class: world.shell.lastErrorClass,
132+
message: "Failed",
133+
...(world.shell.usageLimitResetAt == null
134+
? {}
135+
: { resetAt: world.shell.usageLimitResetAt }),
136+
},
137+
},
138+
],
139+
}))) as never,
114140
receiptStatus: (commandId) => Effect.sync(() => world.receipts.get(commandId) ?? null),
115141
isProviderInstanceEnabledForProject: ({ instanceId }) =>
116142
Effect.sync(() => !world.disabledForProject.has(instanceId)),
117143
dispatch: (command) =>
118144
Effect.suspend(() => {
119-
const target = "modelSelection" in command ? command.modelSelection : undefined;
145+
const target =
146+
command.type === "thread.model-selection.set" || command.type === "provider.switch"
147+
? command.modelSelection
148+
: undefined;
120149
if (target !== undefined) world.selects.push(command.commandId + " " + target.model);
121150
if (target !== undefined && world.refuse.has(target.model)) {
122151
world.receipts.set(command.commandId, "rejected");
@@ -148,7 +177,7 @@ const summary = (commands: ReadonlyArray<OrchestrationV2ServerCommand>) =>
148177
commands.map((command) =>
149178
command.type === "thread.notice.record"
150179
? "notice: " + command.message
151-
: "modelSelection" in command && command.modelSelection !== undefined
180+
: command.type === "thread.model-selection.set" || command.type === "provider.switch"
152181
? command.type + " " + command.modelSelection.model
153182
: command.type,
154183
);
@@ -161,6 +190,79 @@ const withClock = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
161190
TestClock.setTime(NOW).pipe(Effect.andThen(effect));
162191

163192
describe("Fallback.continueThread", () => {
193+
it.effect(
194+
"does not switch a successfully continued run again after restart or model changes",
195+
() => {
196+
const world = makeWorld();
197+
return withClock(
198+
Effect.gen(function* () {
199+
assert.isTrue(yield* continueOnce.pipe(Effect.provide(layer(world, threadChain))));
200+
const commands = [...world.commands];
201+
world.shell = shell();
202+
assert.isTrue(
203+
yield* continueOnce.pipe(
204+
Effect.provide(
205+
layer(world, {
206+
fallbackChains: {
207+
replacement: { steps: [opus, sol], scope: "goals_and_threads" },
208+
},
209+
}),
210+
),
211+
),
212+
);
213+
assert.deepEqual(world.commands, commands);
214+
}),
215+
);
216+
},
217+
);
218+
219+
it.effect("does not continue a run reached by Stop", () => {
220+
const world = makeWorld({ stopped: true });
221+
return withClock(
222+
continueOnce.pipe(
223+
Effect.map((switched) => {
224+
assert.isFalse(switched);
225+
assert.deepEqual(world.commands, []);
226+
}),
227+
Effect.provide(layer(world, threadChain)),
228+
),
229+
);
230+
});
231+
232+
it.effect.each(["provider_error", "transport_error", "unknown", "permission_error"] as const)(
233+
"only switches explicitly configured failure classes: %s",
234+
(failure) =>
235+
withClock(
236+
Effect.gen(function* () {
237+
for (const on of [undefined, ["provider_error", "transport_error"] as const]) {
238+
const world = makeWorld({
239+
shell: shell({ lastErrorClass: failure, usageLimitResetAt: null }),
240+
});
241+
const switched = yield* continueOnce.pipe(
242+
Effect.provide(
243+
layer(world, {
244+
fallbackChains: {
245+
selected: {
246+
steps: [opus, opus2],
247+
scope: "goals_and_threads",
248+
...(on ? { on } : {}),
249+
},
250+
},
251+
}),
252+
),
253+
);
254+
const expected =
255+
on !== undefined && (failure === "provider_error" || failure === "transport_error");
256+
assert.equal(switched, expected);
257+
assert.equal(world.selects.length, expected ? 1 : 0);
258+
assert.equal(
259+
world.commands.filter((command) => command.type === "message.dispatch").length,
260+
expected ? 1 : 0,
261+
);
262+
}
263+
}),
264+
),
265+
);
164266
it.effect(
165267
"switches a usage-limited thread to the next step with a notice and a continuation",
166268
() => {

‎apps/server/src/fleet/resources/Fallback.ts‎

Lines changed: 61 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,12 @@ import {
2121
type ModelSelection,
2222
type ThreadId,
2323
} from "@goat/contracts";
24-
import { modelSelectionCommandType } from "@goat/shared/model";
24+
import { modelSelectionCommandType, modelSelectionsEqual } from "@goat/shared/model";
25+
import {
26+
latestExecutedRun,
27+
latestRootProviderFailure,
28+
threadErrorSummary,
29+
} from "@goat/shared/orchestrationV2ThreadError";
2530
import * as Context from "effect/Context";
2631
import * as DateTime from "effect/DateTime";
2732
import * as Effect from "effect/Effect";
@@ -32,6 +37,7 @@ import * as Schema from "effect/Schema";
3237

3338
import { isExhausted } from "../../channels/channelPlacement.ts";
3439
import * as ProviderSwitch from "../../orchestration-v2/ProviderSwitchService.ts";
40+
import * as Orchestrator from "../../orchestration-v2/Orchestrator.ts";
3541
import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts";
3642
import * as ProviderRegistry from "../../provider/ProviderRegistry.ts";
3743
import * as ServerSettings from "../../serverSettings.ts";
@@ -75,7 +81,7 @@ export interface FallbackShape {
7581
/** Whether any chain opted into ordinary threads, so the recovery sweep looks for them. */
7682
readonly coversThreads: Effect.Effect<boolean>;
7783
/**
78-
* Moves an ordinary thread that failed with a usage limit to its chain's next model, with a
84+
* Moves an ordinary thread with a configured failure trigger to its chain's next model, with a
7985
* notice and a continuation. True when it switched; false leaves the thread to the recovery
8086
* sweep's wait-for-reset.
8187
*/
@@ -105,7 +111,7 @@ const MAX_NOTICED_RUNS = 1000;
105111
const label = (selection: Pick<ModelSelection, "instanceId" | "model">) =>
106112
selection.instanceId + "/" + selection.model;
107113

108-
const make = Effect.gen(function* () {
114+
export const make = Effect.gen(function* () {
109115
const settingsService = yield* ServerSettings.ServerSettingsService;
110116
const threads = yield* ThreadManagement.ThreadManagementService;
111117
const registry = yield* Effect.serviceOption(ProviderRegistry.ProviderRegistry);
@@ -271,23 +277,50 @@ const make = Effect.gen(function* () {
271277
const shell = yield* threads.getThreadShell(threadId);
272278
if (
273279
shell === null ||
274-
shell.status !== "failed" ||
275-
shell.lastErrorClass == null ||
276-
shell.latestRunId === null ||
277280
shell.archivedAt !== null ||
278281
shell.deletedAt !== null ||
279282
shell.settledOverride === "settled" ||
280283
shell.pendingRuntimeRequest != null ||
281284
(yield* snoozed(shell))
282285
)
283286
return false;
284-
const runId = shell.latestRunId;
285-
const failure = shell.lastErrorClass;
286287
const { runs } = yield* threads.getThreadRecords(threadId, ["runs"]);
287-
// The failed run's model, not the thread's: a restart after the selection command
288-
// already moved the thread finds the same step and replays the same command ids.
289-
const failedRun = runs.find((run) => run.id === runId);
290-
if (failedRun === undefined) return false;
288+
const failedRun = latestExecutedRun(runs);
289+
if (failedRun?.status !== "failed") return false;
290+
const runId = failedRun.id;
291+
const continuation = "fleet-fallback:" + threadId + ":" + runId + ":continue";
292+
// A completed hop owns this run even if the chain or selection changed since delivery.
293+
if ((yield* threads.receiptStatus(CommandId.make(continuation))) === "accepted") return true;
294+
if (runs.some((run) => run.status === "queued")) return false;
295+
const { turnItems, providerSessions } = yield* threads.getThreadRecords(
296+
threadId,
297+
["turnItems", "providerSessions"],
298+
{
299+
turnItemTypes: ["error", "run_interrupt_request"],
300+
turnItemRunIds: [runId],
301+
},
302+
);
303+
if (
304+
turnItems.some((item) => item.type === "run_interrupt_request") ||
305+
(shell.limitRecovery?.runId === runId &&
306+
!shell.limitRecovery.autoResume &&
307+
!shell.limitRecovery.snooze)
308+
)
309+
return false;
310+
const session = providerSessions
311+
.filter((entry) => entry.providerInstanceId === shell.providerInstanceId)
312+
.toSorted(
313+
(left, right) =>
314+
DateTime.toEpochMillis(right.updatedAt) - DateTime.toEpochMillis(left.updatedAt) ||
315+
right.id.localeCompare(left.id),
316+
)[0];
317+
const summary = threadErrorSummary(
318+
latestRootProviderFailure(failedRun, turnItems),
319+
session?.lastError ?? null,
320+
);
321+
const failure = summary.lastErrorClass;
322+
if (failure === null) return false;
323+
// A restart after selection must resolve the original chain and replay that hop's receipts.
291324
const current = failedRun.modelSelection;
292325
const resolved = ordinaryThreadChain(settings, current);
293326
if (resolved === null) return false;
@@ -302,15 +335,17 @@ const make = Effect.gen(function* () {
302335
// This read skips the notice; automatedContinuationOfRunId makes the orchestrator
303336
// refuse a continuation that loses the race after it.
304337
const latest = yield* threads.getThreadShell(threadId);
338+
const latestRuns = yield* threads.getThreadRecords(threadId, ["runs"]);
339+
const latestExecuted = latestExecutedRun(latestRuns.runs);
305340
if (
306341
latest === null ||
307-
latest.latestRunId !== runId ||
308-
latest.status !== "failed" ||
342+
latestExecuted?.id !== runId ||
343+
latestExecuted.status !== "failed" ||
309344
latest.archivedAt !== null ||
310345
latest.settledOverride === "settled" ||
311346
latest.pendingRuntimeRequest != null ||
312347
(yield* snoozed(latest)) ||
313-
stepKey(latest.modelSelection) !== stepKey(target)
348+
!modelSelectionsEqual(latest.modelSelection, target)
314349
)
315350
return;
316351
yield* Effect.logInfo("fleet.fallback.hop", {
@@ -329,20 +364,20 @@ const make = Effect.gen(function* () {
329364
failure +
330365
" on " +
331366
label(current) +
332-
(shell.usageLimitResetAt == null
367+
(summary.usageLimitResetAt == null
333368
? ""
334-
: "; that limit resets at " + shell.usageLimitResetAt) +
369+
: "; that limit resets at " + summary.usageLimitResetAt) +
335370
" (fallback chain " +
336371
resolved.id +
337372
").",
338373
);
339-
const continuation = "fleet-fallback:" + threadId + ":" + runId + ":continue";
340374
yield* threads.dispatch({
341375
type: "message.dispatch",
342376
commandId: CommandId.make(continuation),
343377
messageId: MessageId.make(continuation),
344378
threadId,
345379
automatedContinuationOfRunId: runId,
380+
modelSelection: target,
346381
text: "Continue where you left off.",
347382
attachments: [],
348383
dispatchMode: { type: "start_immediately" },
@@ -369,10 +404,11 @@ const make = Effect.gen(function* () {
369404
// A restart after our selection landed: finish that hop instead of choosing again, even
370405
// if its account has since run out. Only our accepted receipt counts; a model the user
371406
// picked by hand goes to the recovery sweep as it would without a chain.
372-
if (stepKey(shell.modelSelection) !== stepKey(current)) {
407+
if (!modelSelectionsEqual(shell.modelSelection, current)) {
373408
const ours =
374-
resolved.chain.steps.some((step) => stepKey(step) === stepKey(shell.modelSelection)) &&
375-
(yield* attempts(shell.modelSelection))._tag === "Accepted";
409+
resolved.chain.steps.some((step) =>
410+
modelSelectionsEqual(toSelection(step), shell.modelSelection),
411+
) && (yield* attempts(shell.modelSelection))._tag === "Accepted";
376412
if (!ours) return false;
377413
yield* announce(shell.modelSelection, new Set(tried.map(stepKey)).size);
378414
return true;
@@ -419,6 +455,10 @@ const make = Effect.gen(function* () {
419455
modelSelection: target,
420456
})
421457
.pipe(
458+
Effect.provideService(Orchestrator.FallbackSelectionSource, {
459+
runId,
460+
modelSelection: current,
461+
}),
422462
Effect.as(true),
423463
Effect.catchCause((cause) =>
424464
Effect.logWarning("fleet.fallback.select-refused", { threadId, cause }).pipe(

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -376,6 +376,17 @@ export interface OrchestratorV2DispatchResult {
376376
readonly storedEvents: ReadonlyArray<OrchestrationV2StoredEvent>;
377377
}
378378

379+
/** Server-only precondition checked under the command lock before fallback changes a selection. */
380+
export const FallbackSelectionSource = Context.Reference<
381+
| {
382+
readonly runId: RunId;
383+
readonly modelSelection: ModelSelection;
384+
}
385+
| undefined
386+
>("goat/orchestration-v2/Orchestrator/FallbackSelectionSource", {
387+
defaultValue: () => undefined,
388+
});
389+
379390
export interface OrchestratorV2Shape {
380391
readonly resumeQueuedRuns: Effect.Effect<number, OrchestratorV2Error>;
381392
readonly recoverDelegatedTasks: Effect.Effect<void>;
@@ -2963,6 +2974,33 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
29632974
cause: `Thread ${command.threadId} is deleted.`,
29642975
});
29652976
}
2977+
if (command.type === "thread.model-selection.set" || command.type === "provider.switch") {
2978+
const source = yield* FallbackSelectionSource;
2979+
if (source !== undefined) {
2980+
const { runs, runtimeRequests } = yield* projectionStore
2981+
.getThreadRecords(command.threadId, ["runs", "runtimeRequests"])
2982+
.pipe(mapDispatchError(command));
2983+
const latest = latestExecutedRun(runs);
2984+
const now = yield* DateTime.now;
2985+
if (
2986+
latest?.id !== source.runId ||
2987+
latest.status !== "failed" ||
2988+
!modelSelectionsEqual(thread.modelSelection, source.modelSelection) ||
2989+
(yield* stopReachedRun(command, command.threadId, source.runId)) ||
2990+
runs.some((run) => run.status === "queued") ||
2991+
runtimeRequests.some((request) => request.status === "pending") ||
2992+
thread.archivedAt !== null ||
2993+
thread.settledOverride === "settled" ||
2994+
(thread.snoozedUntil != null &&
2995+
DateTime.toEpochMillis(thread.snoozedUntil) > DateTime.toEpochMillis(now))
2996+
)
2997+
return yield* new OrchestratorDispatchError({
2998+
commandId: command.commandId,
2999+
commandType: command.type,
3000+
cause: "The fallback source changed before model selection.",
3001+
});
3002+
}
3003+
}
29663004
if (
29673005
command.type === "thread.pull-request.watch" &&
29683006
command.watching &&
@@ -5230,6 +5268,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
52305268
const latest = latestExecutedRun(projection.runs);
52315269
if (
52325270
latest?.id !== command.automatedContinuationOfRunId ||
5271+
(yield* stopReachedRun(command, command.threadId, latest.id)) ||
5272+
(command.modelSelection !== undefined &&
5273+
!modelSelectionsEqual(command.modelSelection, projection.thread.modelSelection)) ||
52335274
latest.status !== "failed" ||
52345275
projection.runs.some((run) => run.status === "queued") ||
52355276
projection.thread.archivedAt !== null ||
@@ -10410,7 +10451,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1041010451
readonly now: DateTime.Utc;
1041110452
}) {
1041210453
const { command, events, thread, run, now } = input;
10413-
if (run.rootNodeId === null || run.providerThreadId === null) return;
10454+
// Startup failures can be recoverable before a provider thread has attached.
10455+
if (run.rootNodeId === null) return;
1041410456
const pendingItems = (yield* Ref.get(events)).flatMap((event) =>
1041510457
event.type === "turn-item.updated" ? [event.payload] : [],
1041610458
);

0 commit comments

Comments
 (0)