77 type OrchestrationV2ConversationMessage ,
88 type OrchestrationV2ExecutionNode ,
99 type OrchestrationV2ProviderCapabilities ,
10+ type OrchestrationV2ProviderRetry ,
1011 type OrchestrationV2ProviderSession ,
1112 type OrchestrationV2ProviderThread ,
1213 type OrchestrationV2ProviderTurn ,
@@ -26,6 +27,7 @@ import {
2627import type { MuseSettings } from "../settings.ts" ;
2728import { AgentScope } from "@t3tools/shared/AgentScope" ;
2829import { getModelSelectionStringOptionValue } from "@t3tools/shared/model" ;
30+ import { formatDuration } from "@t3tools/shared/orchestrationTiming" ;
2931import * as Cause from "effect/Cause" ;
3032import * as DateTime from "effect/DateTime" ;
3133import * as Deferred from "effect/Deferred" ;
@@ -52,7 +54,7 @@ import {
5254 MuseSubscriptionUsage ,
5355 MuseTokenUsageEvent ,
5456 MuseTodoList ,
55- MuseTurnRetryScheduled ,
57+ MuseRetryScheduled ,
5658 MuseTurnCompleted ,
5759 MuseTurnStartResult ,
5860 MuseUserInput ,
@@ -84,7 +86,10 @@ import {
8486 type BackgroundWorkReport ,
8587} from "@t3tools/provider-core/server/notification" ;
8688import type * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests" ;
87- import { makeProviderFailure } from "@t3tools/provider-core/server/failure" ;
89+ import {
90+ makeProviderFailure ,
91+ makeProviderRetryTurnItem ,
92+ } from "@t3tools/provider-core/server/failure" ;
8893import {
8994 makeSubagentChildThread ,
9095 makeSubagentConversationArtifacts ,
@@ -214,6 +219,7 @@ interface ActiveTurn {
214219 readonly started : Map < string , DateTime . Utc > ;
215220 readonly dirty : Set < string > ;
216221 readonly settledRequests : Set < string > ;
222+ readonly retries : Array < MuseRetryRow > ;
217223 readonly done : Deferred . Deferred < void > ;
218224 todoPlanId ?: PlanId ;
219225 nextOrdinal : number ;
@@ -222,6 +228,15 @@ interface ActiveTurn {
222228 compact : boolean ;
223229}
224230
231+ interface MuseRetryRow {
232+ readonly method : string ;
233+ readonly nativeId : string ;
234+ readonly retry : OrchestrationV2ProviderRetry ;
235+ readonly cause : string ;
236+ readonly code : string | undefined ;
237+ running : boolean ;
238+ }
239+
225240interface PendingRequest {
226241 readonly native :
227242 | { type : "approval" ; value : MuseApproval }
@@ -258,9 +273,12 @@ const INFORMATIONAL_NOTIFICATIONS = new Set([
258273 "session/tokenUsage" ,
259274 "session/todoListChanged" ,
260275 "turn/retryScheduled" ,
276+ "turn/modelRetryScheduled" ,
261277 "usage/changed" ,
262278] ) ;
263279const TOOLS_WITH_NATIVE_ROWS = new Set ( [ "write_todos" , "request_user_input" , "workflow" ] ) ;
280+ // Reminder children and other runtime items can start while the model is still retrying.
281+ const MODEL_OUTPUT_KINDS = new Set ( [ "agentMessage" , "reasoning" , "toolCall" ] ) ;
264282const responseAnswerSchema = Schema . Union ( [ Schema . String , Schema . Array ( Schema . String ) ] ) ;
265283
266284export const makeMuseAdapterV2 = Effect . fn ( "makeMuseAdapterV2" ) ( function * (
@@ -447,6 +465,52 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
447465 updatedAt : time ,
448466 } ;
449467 } ;
468+ const emitRetry = Effect . fnUntraced ( function * (
469+ turn : ActiveTurn ,
470+ row : MuseRetryRow ,
471+ status : "running" | "completed" | "interrupted" ,
472+ ) {
473+ const time = yield * DateTime . now ;
474+ const base = baseItem ( turn , row . nativeId , time ) ;
475+ yield * emit ( {
476+ type : "turn_item.updated" ,
477+ driver : MUSE_PROVIDER ,
478+ turnItem : {
479+ ...makeProviderRetryTurnItem ( {
480+ ...base ,
481+ driver : MUSE_PROVIDER ,
482+ nodeId : turn . input . rootNodeId ,
483+ itemOrdinal : base . ordinal ,
484+ failure : makeProviderFailure ( {
485+ // Clients show only the failure message, so a waiting row carries its delay.
486+ message :
487+ status === "running"
488+ ? `${ row . cause } . Retrying in ${ formatDuration ( row . retry . retryDelayMs ?? 0 ) } .`
489+ : row . cause ,
490+ code : row . code ,
491+ class : "provider_error" ,
492+ retryable : true ,
493+ } ) ,
494+ retry : row . retry ,
495+ status,
496+ updatedAt : time ,
497+ } ) ,
498+ // The shared helper keys one row per turn; a long Muse turn can retry several times.
499+ id : base . id ,
500+ } ,
501+ } ) ;
502+ } ) ;
503+ // Muse sends no recovered event, so model output or the turn's end closes open retry rows.
504+ const settleRetries = Effect . fnUntraced ( function * (
505+ turn : ActiveTurn ,
506+ status : "completed" | "interrupted" ,
507+ ) {
508+ for ( const row of turn . retries ) {
509+ if ( ! row . running ) continue ;
510+ row . running = false ;
511+ yield * emitRetry ( turn , row , status ) ;
512+ }
513+ } ) ;
450514 const childItemIdentity = ( context : ChildContext , nativeId : string ) =>
451515 `${ options . instanceId } :child:${ context . childSessionId } :${ nativeId } ` ;
452516 const itemStatus = ( item : MuseItem ) : OrchestrationV2TurnItemStatus => {
@@ -970,6 +1034,8 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
9701034 yield * publishItem ( turn , item , item . status === "inProgress" ? status : undefined ) ;
9711035 }
9721036 turn . dirty . clear ( ) ;
1037+ // A failed turn has its own failure row; its retry rows only record that retrying stopped.
1038+ yield * settleRetries ( turn , status === "completed" ? "completed" : "interrupted" ) ;
9731039 for ( const entry of pending . values ( ) )
9741040 if ( entry . request . providerTurnId !== null ) yield * resolvePending ( entry , "cancelled" ) ;
9751041 turn . providerTurn = { ...turn . providerTurn , status, completedAt } ;
@@ -1413,6 +1479,8 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
14131479 return ;
14141480 const previous = turn . items . get ( item . itemId ) ;
14151481 if ( previous && previous . revision >= item . revision ) return ;
1482+ if ( method === "item/started" && MODEL_OUTPUT_KINDS . has ( item . kind ) )
1483+ yield * settleRetries ( turn , "completed" ) ;
14161484 turn . items . set ( item . itemId , item ) ;
14171485 turn . dirty . delete ( item . itemId ) ;
14181486 yield * publishItem ( turn , item ) ;
@@ -1507,6 +1575,7 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
15071575 break ;
15081576 case "session/tokenUsage" : {
15091577 const usage = yield * decode ( MuseTokenUsageEvent , params ) ;
1578+ yield * settleRetries ( turn , "completed" ) ;
15101579 const previous = turn . providerTurn . turnTokenUsage ;
15111580 const contextUsage = thread ?. contextUsage ;
15121581 turn . providerTurn = {
@@ -1630,21 +1699,32 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
16301699 } ) ;
16311700 break ;
16321701 }
1633- case "turn/retryScheduled" : {
1634- const retry = yield * decode ( MuseTurnRetryScheduled , params ) ;
1635- const time = yield * DateTime . now ;
1636- yield * emit ( {
1637- type : "turn_item.updated" ,
1638- driver : MUSE_PROVIDER ,
1639- turnItem : {
1640- ...baseItem ( turn , `retry:${ retry . nextAttempt } ` , time ) ,
1641- type : "system_notice" ,
1642- status : "completed" ,
1643- title : "Muse retry" ,
1644- completedAt : time ,
1645- message : `Muse is retrying this turn (attempt ${ retry . nextAttempt } of ${ retry . maxAttempts } ): ${ retry . reason } ` ,
1702+ case "turn/retryScheduled" :
1703+ case "turn/modelRetryScheduled" : {
1704+ const retry = yield * decode ( MuseRetryScheduled , params ) ;
1705+ const cause =
1706+ retry . httpStatus === undefined ? retry . errorKind : `HTTP ${ retry . httpStatus } ` ;
1707+ // A chain's first retry opens a row; its later attempts update that row.
1708+ const index =
1709+ retry . attempt > 1 ? turn . retries . findLastIndex ( ( row ) => row . method === method ) : - 1 ;
1710+ const row : MuseRetryRow = {
1711+ method,
1712+ nativeId : turn . retries [ index ] ?. nativeId ?? `retry:${ retry . sourceRange . first . id } ` ,
1713+ // Muse counts the first request as attempt 1; T3 counts retries.
1714+ retry : {
1715+ attempt : retry . attempt ,
1716+ maxAttempts : Math . max ( 1 , retry . maxAttempts - 1 ) ,
1717+ retryDelayMs : Math . trunc ( retry . retryDelayMs ) ,
16461718 } ,
1647- } ) ;
1719+ cause :
1720+ retry . reason ??
1721+ ( cause ? `Model request failed (${ cause } )` : "Model request failed" ) ,
1722+ code : retry . errorKind ,
1723+ running : true ,
1724+ } ;
1725+ if ( index === - 1 ) turn . retries . push ( row ) ;
1726+ else turn . retries [ index ] = row ;
1727+ yield * emitRetry ( turn , row , "running" ) ;
16481728 break ;
16491729 }
16501730 }
@@ -2041,6 +2121,7 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
20412121 started : new Map ( ) ,
20422122 dirty : new Set ( ) ,
20432123 settledRequests : new Set ( ) ,
2124+ retries : [ ] ,
20442125 done : yield * Deferred . make < void > ( ) ,
20452126 nextOrdinal : turnInput . providerTurnOrdinal * 100 + 1 ,
20462127 flushScheduled : false ,
0 commit comments