@@ -21,7 +21,12 @@ import {
2121 type ModelSelection ,
2222 type ThreadId ,
2323} from "@goat/contracts" ;
24- import { modelSelectionCommandType } from "@goat/shared/model" ;
24+ import { modelSelectionCommandType , modelSelectionsEqual } from "@goat/shared/model" ;
25+ import {
26+ latestExecutedRun ,
27+ latestRootProviderFailure ,
28+ threadErrorSummary ,
29+ } from "@goat/shared/orchestrationV2ThreadError" ;
2530import * as Context from "effect/Context" ;
2631import * as DateTime from "effect/DateTime" ;
2732import * as Effect from "effect/Effect" ;
@@ -32,6 +37,7 @@ import * as Schema from "effect/Schema";
3237
3338import { isExhausted } from "../../channels/channelPlacement.ts" ;
3439import * as ProviderSwitch from "../../orchestration-v2/ProviderSwitchService.ts" ;
40+ import * as Orchestrator from "../../orchestration-v2/Orchestrator.ts" ;
3541import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts" ;
3642import * as ProviderRegistry from "../../provider/ProviderRegistry.ts" ;
3743import * as ServerSettings from "../../serverSettings.ts" ;
@@ -75,7 +81,7 @@ export interface FallbackShape {
7581 /** Whether any chain opted into ordinary threads, so the recovery sweep looks for them. */
7682 readonly coversThreads : Effect . Effect < boolean > ;
7783 /**
78- * Moves an ordinary thread that failed with a usage limit to its chain's next model, with a
84+ * Moves an ordinary thread with a configured failure trigger to its chain's next model, with a
7985 * notice and a continuation. True when it switched; false leaves the thread to the recovery
8086 * sweep's wait-for-reset.
8187 */
@@ -105,7 +111,7 @@ const MAX_NOTICED_RUNS = 1000;
105111const label = ( selection : Pick < ModelSelection , "instanceId" | "model" > ) =>
106112 selection . instanceId + "/" + selection . model ;
107113
108- const make = Effect . gen ( function * ( ) {
114+ export const make = Effect . gen ( function * ( ) {
109115 const settingsService = yield * ServerSettings . ServerSettingsService ;
110116 const threads = yield * ThreadManagement . ThreadManagementService ;
111117 const registry = yield * Effect . serviceOption ( ProviderRegistry . ProviderRegistry ) ;
@@ -271,23 +277,50 @@ const make = Effect.gen(function* () {
271277 const shell = yield * threads . getThreadShell ( threadId ) ;
272278 if (
273279 shell === null ||
274- shell . status !== "failed" ||
275- shell . lastErrorClass == null ||
276- shell . latestRunId === null ||
277280 shell . archivedAt !== null ||
278281 shell . deletedAt !== null ||
279282 shell . settledOverride === "settled" ||
280283 shell . pendingRuntimeRequest != null ||
281284 ( yield * snoozed ( shell ) )
282285 )
283286 return false ;
284- const runId = shell . latestRunId ;
285- const failure = shell . lastErrorClass ;
286287 const { runs } = yield * threads . getThreadRecords ( threadId , [ "runs" ] ) ;
287- // The failed run's model, not the thread's: a restart after the selection command
288- // already moved the thread finds the same step and replays the same command ids.
289- const failedRun = runs . find ( ( run ) => run . id === runId ) ;
290- if ( failedRun === undefined ) return false ;
288+ const failedRun = latestExecutedRun ( runs ) ;
289+ if ( failedRun ?. status !== "failed" ) return false ;
290+ const runId = failedRun . id ;
291+ const continuation = "fleet-fallback:" + threadId + ":" + runId + ":continue" ;
292+ // A completed hop owns this run even if the chain or selection changed since delivery.
293+ if ( ( yield * threads . receiptStatus ( CommandId . make ( continuation ) ) ) === "accepted" ) return true ;
294+ if ( runs . some ( ( run ) => run . status === "queued" ) ) return false ;
295+ const { turnItems, providerSessions } = yield * threads . getThreadRecords (
296+ threadId ,
297+ [ "turnItems" , "providerSessions" ] ,
298+ {
299+ turnItemTypes : [ "error" , "run_interrupt_request" ] ,
300+ turnItemRunIds : [ runId ] ,
301+ } ,
302+ ) ;
303+ if (
304+ turnItems . some ( ( item ) => item . type === "run_interrupt_request" ) ||
305+ ( shell . limitRecovery ?. runId === runId &&
306+ ! shell . limitRecovery . autoResume &&
307+ ! shell . limitRecovery . snooze )
308+ )
309+ return false ;
310+ const session = providerSessions
311+ . filter ( ( entry ) => entry . providerInstanceId === shell . providerInstanceId )
312+ . toSorted (
313+ ( left , right ) =>
314+ DateTime . toEpochMillis ( right . updatedAt ) - DateTime . toEpochMillis ( left . updatedAt ) ||
315+ right . id . localeCompare ( left . id ) ,
316+ ) [ 0 ] ;
317+ const summary = threadErrorSummary (
318+ latestRootProviderFailure ( failedRun , turnItems ) ,
319+ session ?. lastError ?? null ,
320+ ) ;
321+ const failure = summary . lastErrorClass ;
322+ if ( failure === null ) return false ;
323+ // A restart after selection must resolve the original chain and replay that hop's receipts.
291324 const current = failedRun . modelSelection ;
292325 const resolved = ordinaryThreadChain ( settings , current ) ;
293326 if ( resolved === null ) return false ;
@@ -302,15 +335,17 @@ const make = Effect.gen(function* () {
302335 // This read skips the notice; automatedContinuationOfRunId makes the orchestrator
303336 // refuse a continuation that loses the race after it.
304337 const latest = yield * threads . getThreadShell ( threadId ) ;
338+ const latestRuns = yield * threads . getThreadRecords ( threadId , [ "runs" ] ) ;
339+ const latestExecuted = latestExecutedRun ( latestRuns . runs ) ;
305340 if (
306341 latest === null ||
307- latest . latestRunId !== runId ||
308- latest . status !== "failed" ||
342+ latestExecuted ?. id !== runId ||
343+ latestExecuted . status !== "failed" ||
309344 latest . archivedAt !== null ||
310345 latest . settledOverride === "settled" ||
311346 latest . pendingRuntimeRequest != null ||
312347 ( yield * snoozed ( latest ) ) ||
313- stepKey ( latest . modelSelection ) !== stepKey ( target )
348+ ! modelSelectionsEqual ( latest . modelSelection , target )
314349 )
315350 return ;
316351 yield * Effect . logInfo ( "fleet.fallback.hop" , {
@@ -329,20 +364,20 @@ const make = Effect.gen(function* () {
329364 failure +
330365 " on " +
331366 label ( current ) +
332- ( shell . usageLimitResetAt == null
367+ ( summary . usageLimitResetAt == null
333368 ? ""
334- : "; that limit resets at " + shell . usageLimitResetAt ) +
369+ : "; that limit resets at " + summary . usageLimitResetAt ) +
335370 " (fallback chain " +
336371 resolved . id +
337372 ")." ,
338373 ) ;
339- const continuation = "fleet-fallback:" + threadId + ":" + runId + ":continue" ;
340374 yield * threads . dispatch ( {
341375 type : "message.dispatch" ,
342376 commandId : CommandId . make ( continuation ) ,
343377 messageId : MessageId . make ( continuation ) ,
344378 threadId,
345379 automatedContinuationOfRunId : runId ,
380+ modelSelection : target ,
346381 text : "Continue where you left off." ,
347382 attachments : [ ] ,
348383 dispatchMode : { type : "start_immediately" } ,
@@ -369,10 +404,11 @@ const make = Effect.gen(function* () {
369404 // A restart after our selection landed: finish that hop instead of choosing again, even
370405 // if its account has since run out. Only our accepted receipt counts; a model the user
371406 // picked by hand goes to the recovery sweep as it would without a chain.
372- if ( stepKey ( shell . modelSelection ) !== stepKey ( current ) ) {
407+ if ( ! modelSelectionsEqual ( shell . modelSelection , current ) ) {
373408 const ours =
374- resolved . chain . steps . some ( ( step ) => stepKey ( step ) === stepKey ( shell . modelSelection ) ) &&
375- ( yield * attempts ( shell . modelSelection ) ) . _tag === "Accepted" ;
409+ resolved . chain . steps . some ( ( step ) =>
410+ modelSelectionsEqual ( toSelection ( step ) , shell . modelSelection ) ,
411+ ) && ( yield * attempts ( shell . modelSelection ) ) . _tag === "Accepted" ;
376412 if ( ! ours ) return false ;
377413 yield * announce ( shell . modelSelection , new Set ( tried . map ( stepKey ) ) . size ) ;
378414 return true ;
@@ -419,6 +455,10 @@ const make = Effect.gen(function* () {
419455 modelSelection : target ,
420456 } )
421457 . pipe (
458+ Effect . provideService ( Orchestrator . FallbackSelectionSource , {
459+ runId,
460+ modelSelection : current ,
461+ } ) ,
422462 Effect . as ( true ) ,
423463 Effect . catchCause ( ( cause ) =>
424464 Effect . logWarning ( "fleet.fallback.select-refused" , { threadId, cause } ) . pipe (
0 commit comments