Skip to content

Commit 84094ed

Browse files
t3dotggclaude
andcommitted
fix(muse): show model retries, and each retry chain gets its own row
Muse sends turn/modelRetryScheduled when a model request fails with an HTTP error such as 504 or 429. The adapter dropped it, so a turn showed one stale "retrying this turn" notice and "Thinking" for minutes. Handle both retry events with one notice that says the delay and the HTTP status. A chain's first retry opens a row keyed by Muse's durable record id; later attempts update that row, so a later chain no longer overwrites an earlier one. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 920ffcd commit 84094ed

3 files changed

Lines changed: 78 additions & 7 deletions

File tree

‎packages/provider-muse/src/server/adapter.test.ts‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1888,6 +1888,57 @@ describe("MuseAdapterV2", () => {
18881888
}).pipe(Effect.scoped, Effect.provide(testLayer)),
18891889
);
18901890

1891+
it.effect("shows each turn and model retry chain on its own row", () =>
1892+
Effect.gen(function* () {
1893+
const fake = yield* makeFakeMuse();
1894+
const harness = yield* makeHarness(fake);
1895+
const { nativeId } = yield* startConversation(harness, fake);
1896+
const retry = Effect.fnUntraced(function* (
1897+
method: string,
1898+
record: string,
1899+
attempt: number,
1900+
params: Record<string, unknown>,
1901+
) {
1902+
yield* fake.emit(method, {
1903+
turnId: nativeId,
1904+
attempt,
1905+
nextAttempt: attempt + 1,
1906+
sourceRange: { first: { id: record }, last: { id: record } },
1907+
...params,
1908+
});
1909+
const { turnItem } = yield* harness.takeEvent(
1910+
"turn_item.updated",
1911+
(event) => event.turnItem.type === "system_notice",
1912+
);
1913+
return turnItem.type === "system_notice" ? turnItem : undefined;
1914+
});
1915+
const timeout = { maxAttempts: 2, retryDelayMs: 1_000, reason: "no first model event" };
1916+
const turnRetry = yield* retry("turn/retryScheduled", "record-1", 1, timeout);
1917+
assert.strictEqual(
1918+
turnRetry?.message,
1919+
"Muse is retrying this turn in 1s (attempt 2 of 2): no first model event",
1920+
);
1921+
const http = { maxAttempts: 10, errorKind: "server", httpStatus: 504, retryDelayMs: 1_000 };
1922+
const modelRetry = yield* retry("turn/modelRetryScheduled", "record-2", 1, http);
1923+
assert.strictEqual(
1924+
modelRetry?.message,
1925+
"Muse is retrying the model request in 1s (attempt 2 of 10): HTTP 504",
1926+
);
1927+
const nextChain = yield* retry("turn/retryScheduled", "record-3", 1, timeout);
1928+
assert.strictEqual(new Set([turnRetry?.id, modelRetry?.id, nextChain?.id]).size, 3);
1929+
const sameChain = yield* retry("turn/modelRetryScheduled", "record-4", 2, {
1930+
...http,
1931+
httpStatus: 429,
1932+
retryDelayMs: 60_000,
1933+
});
1934+
assert.strictEqual(sameChain?.id, modelRetry?.id);
1935+
assert.strictEqual(
1936+
sameChain?.message,
1937+
"Muse is retrying the model request in 1m (attempt 3 of 10): HTTP 429",
1938+
);
1939+
}).pipe(Effect.scoped, Effect.provide(testLayer)),
1940+
);
1941+
18911942
it.effect("opens a prompt-and-result child thread for a native subagent", () =>
18921943
Effect.gen(function* () {
18931944
const fake = yield* makeFakeMuse();

‎packages/provider-muse/src/server/adapter.ts‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import {
2626
import type { MuseSettings } from "../settings.ts";
2727
import { AgentScope } from "@t3tools/shared/AgentScope";
2828
import { getModelSelectionStringOptionValue } from "@t3tools/shared/model";
29+
import { formatDuration } from "@t3tools/shared/orchestrationTiming";
2930
import * as Cause from "effect/Cause";
3031
import * as DateTime from "effect/DateTime";
3132
import * as Deferred from "effect/Deferred";
@@ -52,7 +53,7 @@ import {
5253
MuseSubscriptionUsage,
5354
MuseTokenUsageEvent,
5455
MuseTodoList,
55-
MuseTurnRetryScheduled,
56+
MuseRetryScheduled,
5657
MuseTurnCompleted,
5758
MuseTurnStartResult,
5859
MuseUserInput,
@@ -214,6 +215,8 @@ interface ActiveTurn {
214215
readonly started: Map<string, DateTime.Utc>;
215216
readonly dirty: Set<string>;
216217
readonly settledRequests: Set<string>;
218+
/** The row each retry notification method's current chain updates. */
219+
readonly retryRows: Map<string, string>;
217220
readonly done: Deferred.Deferred<void>;
218221
todoPlanId?: PlanId;
219222
nextOrdinal: number;
@@ -258,6 +261,7 @@ const INFORMATIONAL_NOTIFICATIONS = new Set([
258261
"session/tokenUsage",
259262
"session/todoListChanged",
260263
"turn/retryScheduled",
264+
"turn/modelRetryScheduled",
261265
"usage/changed",
262266
]);
263267
const TOOLS_WITH_NATIVE_ROWS = new Set(["write_todos", "request_user_input", "workflow"]);
@@ -1630,19 +1634,29 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
16301634
});
16311635
break;
16321636
}
1633-
case "turn/retryScheduled": {
1634-
const retry = yield* decode(MuseTurnRetryScheduled, params);
1637+
case "turn/retryScheduled":
1638+
case "turn/modelRetryScheduled": {
1639+
const retry = yield* decode(MuseRetryScheduled, params);
1640+
// A chain's first retry opens a row; its later attempts update that row.
1641+
const row =
1642+
(retry.attempt > 1 ? turn.retryRows.get(method) : undefined) ??
1643+
`retry:${retry.sourceRange.first.id}`;
1644+
turn.retryRows.set(method, row);
1645+
const subject = method === "turn/retryScheduled" ? "this turn" : "the model request";
1646+
const reason =
1647+
retry.reason ??
1648+
(retry.httpStatus === undefined ? retry.errorKind : `HTTP ${retry.httpStatus}`);
16351649
const time = yield* DateTime.now;
16361650
yield* emit({
16371651
type: "turn_item.updated",
16381652
driver: MUSE_PROVIDER,
16391653
turnItem: {
1640-
...baseItem(turn, `retry:${retry.nextAttempt}`, time),
1654+
...baseItem(turn, row, time),
16411655
type: "system_notice",
16421656
status: "completed",
16431657
title: "Muse retry",
16441658
completedAt: time,
1645-
message: `Muse is retrying this turn (attempt ${retry.nextAttempt} of ${retry.maxAttempts}): ${retry.reason}`,
1659+
message: `Muse is retrying ${subject} in ${formatDuration(retry.retryDelayMs)} (attempt ${retry.nextAttempt} of ${retry.maxAttempts})${reason ? `: ${reason}` : ""}`,
16461660
},
16471661
});
16481662
break;
@@ -2041,6 +2055,7 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
20412055
started: new Map(),
20422056
dirty: new Set(),
20432057
settledRequests: new Set(),
2058+
retryRows: new Map(),
20442059
done: yield* Deferred.make<void>(),
20452060
nextOrdinal: turnInput.providerTurnOrdinal * 100 + 1,
20462061
flushScheduled: false,

‎packages/provider-muse/src/server/protocol.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -182,12 +182,17 @@ export const MuseSubscriptionUsage = Schema.Struct({
182182
}),
183183
});
184184
export type MuseSubscriptionUsage = typeof MuseSubscriptionUsage.Type;
185-
export const MuseTurnRetryScheduled = Schema.Struct({
185+
/** `turn/retryScheduled` carries `reason`; `turn/modelRetryScheduled` carries the HTTP failure. */
186+
export const MuseRetryScheduled = Schema.Struct({
186187
turnId: NonEmptyString,
188+
attempt: Schema.Int.check(Schema.isGreaterThan(0)),
187189
nextAttempt: Schema.Int.check(Schema.isGreaterThan(0)),
188190
maxAttempts: Schema.Int.check(Schema.isGreaterThan(0)),
189-
reason: NonEmptyString,
191+
reason: Schema.optional(NonEmptyString),
192+
errorKind: Schema.optional(Schema.String),
193+
httpStatus: Schema.optional(Schema.Int),
190194
retryDelayMs: Schema.Finite.check(Schema.isGreaterThanOrEqualTo(0)),
195+
sourceRange: Schema.Struct({ first: Schema.Struct({ id: NonEmptyString }) }),
191196
});
192197
export const MuseCompactResult = Schema.Struct({
193198
status: Schema.String,

0 commit comments

Comments
 (0)