Skip to content

Commit 7da0ef0

Browse files
t3dotggclaude
andauthored
fix(muse): retries show while Muse waits and fold away once it recovers (#33151)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 0770446 commit 7da0ef0

3 files changed

Lines changed: 187 additions & 18 deletions

File tree

‎packages/provider-muse/src/server/adapter.test.ts‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1888,6 +1888,89 @@ describe("MuseAdapterV2", () => {
18881888
}).pipe(Effect.scoped, Effect.provide(testLayer)),
18891889
);
18901890

1891+
it.effect("keeps each retry chain on one row that settles when the turn moves on", () =>
1892+
Effect.gen(function* () {
1893+
const fake = yield* makeFakeMuse();
1894+
const harness = yield* makeHarness(fake);
1895+
const { nativeId } = yield* startConversation(harness, fake);
1896+
const takeRetryRow = Effect.fnUntraced(function* () {
1897+
const { turnItem } = yield* harness.takeEvent(
1898+
"turn_item.updated",
1899+
(event) => event.turnItem.type === "error",
1900+
);
1901+
return turnItem.type === "error" ? turnItem : undefined;
1902+
});
1903+
const retry = Effect.fnUntraced(function* (
1904+
method: string,
1905+
record: string,
1906+
attempt: number,
1907+
params: Record<string, unknown>,
1908+
) {
1909+
yield* fake.emit(method, {
1910+
turnId: nativeId,
1911+
attempt,
1912+
nextAttempt: attempt + 1,
1913+
sourceRange: { first: { id: record }, last: { id: record } },
1914+
...params,
1915+
});
1916+
return yield* takeRetryRow();
1917+
});
1918+
const timeout = { maxAttempts: 2, retryDelayMs: 1_000, reason: "no first model event" };
1919+
const turnRetry = yield* retry("turn/retryScheduled", "record-1", 1, timeout);
1920+
assert.strictEqual(turnRetry?.status, "running");
1921+
assert.strictEqual(turnRetry?.failure.message, "no first model event. Retrying in 1s.");
1922+
assert.deepStrictEqual(turnRetry?.retry, { attempt: 1, maxAttempts: 1, retryDelayMs: 1_000 });
1923+
const http = { maxAttempts: 10, errorKind: "server", httpStatus: 504, retryDelayMs: 1_000 };
1924+
const modelRetry = yield* retry("turn/modelRetryScheduled", "record-2", 1, http);
1925+
assert.strictEqual(
1926+
modelRetry?.failure.message,
1927+
"Model request failed (HTTP 504). Retrying in 1s.",
1928+
);
1929+
const nextChain = yield* retry("turn/retryScheduled", "record-3", 1, timeout);
1930+
assert.strictEqual(new Set([turnRetry?.id, modelRetry?.id, nextChain?.id]).size, 3);
1931+
const sameChain = yield* retry("turn/modelRetryScheduled", "record-4", 2, {
1932+
...http,
1933+
httpStatus: 429,
1934+
retryDelayMs: 60_000,
1935+
});
1936+
assert.strictEqual(sameChain?.id, modelRetry?.id);
1937+
assert.deepStrictEqual(sameChain?.retry, {
1938+
attempt: 2,
1939+
maxAttempts: 9,
1940+
retryDelayMs: 60_000,
1941+
});
1942+
1943+
yield* fake.emit("item/started", {
1944+
turnId: nativeId,
1945+
item: {
1946+
itemId: "answer",
1947+
turnId: nativeId,
1948+
kind: "agentMessage",
1949+
revision: 1,
1950+
status: "inProgress",
1951+
},
1952+
});
1953+
const recovered = [yield* takeRetryRow(), yield* takeRetryRow(), yield* takeRetryRow()];
1954+
assert.sameMembers(
1955+
recovered.map((row) => `${row?.id} ${row?.status} ${row?.failure.message}`),
1956+
[
1957+
`${turnRetry?.id} completed no first model event`,
1958+
`${nextChain?.id} completed no first model event`,
1959+
`${sameChain?.id} completed Model request failed (HTTP 429)`,
1960+
],
1961+
);
1962+
1963+
const stalled = yield* retry("turn/modelRetryScheduled", "record-5", 1, http);
1964+
yield* fake.emit("turn/completed", {
1965+
turnId: nativeId,
1966+
terminal: "failed",
1967+
error: { message: "Muse gave up" },
1968+
});
1969+
const stopped = yield* takeRetryRow();
1970+
assert.deepStrictEqual([stopped?.id, stopped?.status], [stalled?.id, "interrupted"]);
1971+
}).pipe(Effect.scoped, Effect.provide(testLayer)),
1972+
);
1973+
18911974
it.effect("opens a prompt-and-result child thread for a native subagent", () =>
18921975
Effect.gen(function* () {
18931976
const fake = yield* makeFakeMuse();

‎packages/provider-muse/src/server/adapter.ts‎

Lines changed: 97 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
type OrchestrationV2ConversationMessage,
88
type OrchestrationV2ExecutionNode,
99
type OrchestrationV2ProviderCapabilities,
10+
type OrchestrationV2ProviderRetry,
1011
type OrchestrationV2ProviderSession,
1112
type OrchestrationV2ProviderThread,
1213
type OrchestrationV2ProviderTurn,
@@ -26,6 +27,7 @@ import {
2627
import type { MuseSettings } from "../settings.ts";
2728
import { AgentScope } from "@t3tools/shared/AgentScope";
2829
import { getModelSelectionStringOptionValue } from "@t3tools/shared/model";
30+
import { formatDuration } from "@t3tools/shared/orchestrationTiming";
2931
import * as Cause from "effect/Cause";
3032
import * as DateTime from "effect/DateTime";
3133
import * as Deferred from "effect/Deferred";
@@ -52,7 +54,7 @@ import {
5254
MuseSubscriptionUsage,
5355
MuseTokenUsageEvent,
5456
MuseTodoList,
55-
MuseTurnRetryScheduled,
57+
MuseRetryScheduled,
5658
MuseTurnCompleted,
5759
MuseTurnStartResult,
5860
MuseUserInput,
@@ -84,7 +86,10 @@ import {
8486
type BackgroundWorkReport,
8587
} from "@t3tools/provider-core/server/notification";
8688
import type * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
87-
import { makeProviderFailure } from "@t3tools/provider-core/server/failure";
89+
import {
90+
makeProviderFailure,
91+
makeProviderRetryTurnItem,
92+
} from "@t3tools/provider-core/server/failure";
8893
import {
8994
makeSubagentChildThread,
9095
makeSubagentConversationArtifacts,
@@ -214,6 +219,7 @@ interface ActiveTurn {
214219
readonly started: Map<string, DateTime.Utc>;
215220
readonly dirty: Set<string>;
216221
readonly settledRequests: Set<string>;
222+
readonly retries: Array<MuseRetryRow>;
217223
readonly done: Deferred.Deferred<void>;
218224
todoPlanId?: PlanId;
219225
nextOrdinal: number;
@@ -222,6 +228,15 @@ interface ActiveTurn {
222228
compact: boolean;
223229
}
224230

231+
interface MuseRetryRow {
232+
readonly method: string;
233+
readonly nativeId: string;
234+
readonly retry: OrchestrationV2ProviderRetry;
235+
readonly cause: string;
236+
readonly code: string | undefined;
237+
running: boolean;
238+
}
239+
225240
interface PendingRequest {
226241
readonly native:
227242
| { type: "approval"; value: MuseApproval }
@@ -258,9 +273,12 @@ const INFORMATIONAL_NOTIFICATIONS = new Set([
258273
"session/tokenUsage",
259274
"session/todoListChanged",
260275
"turn/retryScheduled",
276+
"turn/modelRetryScheduled",
261277
"usage/changed",
262278
]);
263279
const TOOLS_WITH_NATIVE_ROWS = new Set(["write_todos", "request_user_input", "workflow"]);
280+
// Reminder children and other runtime items can start while the model is still retrying.
281+
const MODEL_OUTPUT_KINDS = new Set(["agentMessage", "reasoning", "toolCall"]);
264282
const responseAnswerSchema = Schema.Union([Schema.String, Schema.Array(Schema.String)]);
265283

266284
export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
@@ -447,6 +465,52 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
447465
updatedAt: time,
448466
};
449467
};
468+
const emitRetry = Effect.fnUntraced(function* (
469+
turn: ActiveTurn,
470+
row: MuseRetryRow,
471+
status: "running" | "completed" | "interrupted",
472+
) {
473+
const time = yield* DateTime.now;
474+
const base = baseItem(turn, row.nativeId, time);
475+
yield* emit({
476+
type: "turn_item.updated",
477+
driver: MUSE_PROVIDER,
478+
turnItem: {
479+
...makeProviderRetryTurnItem({
480+
...base,
481+
driver: MUSE_PROVIDER,
482+
nodeId: turn.input.rootNodeId,
483+
itemOrdinal: base.ordinal,
484+
failure: makeProviderFailure({
485+
// Clients show only the failure message, so a waiting row carries its delay.
486+
message:
487+
status === "running"
488+
? `${row.cause}. Retrying in ${formatDuration(row.retry.retryDelayMs ?? 0)}.`
489+
: row.cause,
490+
code: row.code,
491+
class: "provider_error",
492+
retryable: true,
493+
}),
494+
retry: row.retry,
495+
status,
496+
updatedAt: time,
497+
}),
498+
// The shared helper keys one row per turn; a long Muse turn can retry several times.
499+
id: base.id,
500+
},
501+
});
502+
});
503+
// Muse sends no recovered event, so model output or the turn's end closes open retry rows.
504+
const settleRetries = Effect.fnUntraced(function* (
505+
turn: ActiveTurn,
506+
status: "completed" | "interrupted",
507+
) {
508+
for (const row of turn.retries) {
509+
if (!row.running) continue;
510+
row.running = false;
511+
yield* emitRetry(turn, row, status);
512+
}
513+
});
450514
const childItemIdentity = (context: ChildContext, nativeId: string) =>
451515
`${options.instanceId}:child:${context.childSessionId}:${nativeId}`;
452516
const itemStatus = (item: MuseItem): OrchestrationV2TurnItemStatus => {
@@ -970,6 +1034,8 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
9701034
yield* publishItem(turn, item, item.status === "inProgress" ? status : undefined);
9711035
}
9721036
turn.dirty.clear();
1037+
// A failed turn has its own failure row; its retry rows only record that retrying stopped.
1038+
yield* settleRetries(turn, status === "completed" ? "completed" : "interrupted");
9731039
for (const entry of pending.values())
9741040
if (entry.request.providerTurnId !== null) yield* resolvePending(entry, "cancelled");
9751041
turn.providerTurn = { ...turn.providerTurn, status, completedAt };
@@ -1413,6 +1479,8 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
14131479
return;
14141480
const previous = turn.items.get(item.itemId);
14151481
if (previous && previous.revision >= item.revision) return;
1482+
if (method === "item/started" && MODEL_OUTPUT_KINDS.has(item.kind))
1483+
yield* settleRetries(turn, "completed");
14161484
turn.items.set(item.itemId, item);
14171485
turn.dirty.delete(item.itemId);
14181486
yield* publishItem(turn, item);
@@ -1507,6 +1575,7 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
15071575
break;
15081576
case "session/tokenUsage": {
15091577
const usage = yield* decode(MuseTokenUsageEvent, params);
1578+
yield* settleRetries(turn, "completed");
15101579
const previous = turn.providerTurn.turnTokenUsage;
15111580
const contextUsage = thread?.contextUsage;
15121581
turn.providerTurn = {
@@ -1630,21 +1699,32 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
16301699
});
16311700
break;
16321701
}
1633-
case "turn/retryScheduled": {
1634-
const retry = yield* decode(MuseTurnRetryScheduled, params);
1635-
const time = yield* DateTime.now;
1636-
yield* emit({
1637-
type: "turn_item.updated",
1638-
driver: MUSE_PROVIDER,
1639-
turnItem: {
1640-
...baseItem(turn, `retry:${retry.nextAttempt}`, time),
1641-
type: "system_notice",
1642-
status: "completed",
1643-
title: "Muse retry",
1644-
completedAt: time,
1645-
message: `Muse is retrying this turn (attempt ${retry.nextAttempt} of ${retry.maxAttempts}): ${retry.reason}`,
1702+
case "turn/retryScheduled":
1703+
case "turn/modelRetryScheduled": {
1704+
const retry = yield* decode(MuseRetryScheduled, params);
1705+
const cause =
1706+
retry.httpStatus === undefined ? retry.errorKind : `HTTP ${retry.httpStatus}`;
1707+
// A chain's first retry opens a row; its later attempts update that row.
1708+
const index =
1709+
retry.attempt > 1 ? turn.retries.findLastIndex((row) => row.method === method) : -1;
1710+
const row: MuseRetryRow = {
1711+
method,
1712+
nativeId: turn.retries[index]?.nativeId ?? `retry:${retry.sourceRange.first.id}`,
1713+
// Muse counts the first request as attempt 1; T3 counts retries.
1714+
retry: {
1715+
attempt: retry.attempt,
1716+
maxAttempts: Math.max(1, retry.maxAttempts - 1),
1717+
retryDelayMs: Math.trunc(retry.retryDelayMs),
16461718
},
1647-
});
1719+
cause:
1720+
retry.reason ??
1721+
(cause ? `Model request failed (${cause})` : "Model request failed"),
1722+
code: retry.errorKind,
1723+
running: true,
1724+
};
1725+
if (index === -1) turn.retries.push(row);
1726+
else turn.retries[index] = row;
1727+
yield* emitRetry(turn, row, "running");
16481728
break;
16491729
}
16501730
}
@@ -2041,6 +2121,7 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
20412121
started: new Map(),
20422122
dirty: new Set(),
20432123
settledRequests: new Set(),
2124+
retries: [],
20442125
done: yield* Deferred.make<void>(),
20452126
nextOrdinal: turnInput.providerTurnOrdinal * 100 + 1,
20462127
flushScheduled: false,

‎packages/provider-muse/src/server/protocol.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -182,12 +182,17 @@ export const MuseSubscriptionUsage = Schema.Struct({
182182
}),
183183
});
184184
export type MuseSubscriptionUsage = typeof MuseSubscriptionUsage.Type;
185-
export const MuseTurnRetryScheduled = Schema.Struct({
185+
/** `turn/retryScheduled` carries `reason`; `turn/modelRetryScheduled` carries the HTTP failure. */
186+
export const MuseRetryScheduled = Schema.Struct({
186187
turnId: NonEmptyString,
188+
attempt: Schema.Int.check(Schema.isGreaterThan(0)),
187189
nextAttempt: Schema.Int.check(Schema.isGreaterThan(0)),
188190
maxAttempts: Schema.Int.check(Schema.isGreaterThan(0)),
189-
reason: NonEmptyString,
191+
reason: Schema.optional(NonEmptyString),
192+
errorKind: Schema.optional(Schema.String),
193+
httpStatus: Schema.optional(Schema.Int),
190194
retryDelayMs: Schema.Finite.check(Schema.isGreaterThanOrEqualTo(0)),
195+
sourceRange: Schema.Struct({ first: Schema.Struct({ id: NonEmptyString }) }),
191196
});
192197
export const MuseCompactResult = Schema.Struct({
193198
status: Schema.String,

0 commit comments

Comments
 (0)