Skip to content

Commit 93ea95c

Browse files
ntindleclaude
andauthored
fix(server): Muse turns stopped by the subscription quota are Limited, not failed (pingdotgg#18266)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent ddcde11 commit 93ea95c

5 files changed

Lines changed: 318 additions & 5 deletions

File tree

‎packages/provider-muse/src/server/adapter.test.ts‎

Lines changed: 143 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ import * as Stream from "effect/Stream";
3333

3434
import * as TestProviderHost from "@t3tools/provider-testing/TestProviderHost";
3535
import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
36-
import type { MuseItem } from "./protocol.ts";
36+
import type { MuseItem, MuseSubscriptionUsage } from "./protocol.ts";
3737
import type { MuseSdkHost } from "./sdk.ts";
3838
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
3939
import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
@@ -173,6 +173,11 @@ const makeFakeMuse = Effect.fnUntraced(function* (idPrefix = "native") {
173173
Effect.sync(() => {
174174
notify({ jsonrpc: "2.0", method, params: { sessionId, ...params } });
175175
}),
176+
/** A notification about the host or account rather than a session, as `usage/changed` is. */
177+
emitHostWide: (method: string, params: Record<string, unknown>) =>
178+
Effect.sync(() => {
179+
notify({ jsonrpc: "2.0", method, params });
180+
}),
176181
takeCall: Effect.fnUntraced(function* (method: string) {
177182
while (true) {
178183
const call = yield* Queue.take(requests);
@@ -339,6 +344,20 @@ const startConversation = Effect.fnUntraced(function* (
339344
return { nativeId: call.commandId!, providerTurn: event.providerTurn };
340345
});
341346

347+
// `usage/changed` as Muse 1.4.3 sent it after a reply (2026-10-08), with a placeholder tier.
348+
const SUBSCRIPTION_USAGE = {
349+
window: { usedPercent: 0, windowDurationMins: 300, resetsAtMs: 1791465557000 },
350+
weekly: { usedPercent: 5, resetsAtMs: 1791763200000 },
351+
tier: "tier-1",
352+
observedAtMs: 1791447762610,
353+
} satisfies MuseSubscriptionUsage;
354+
// How Muse ends a turn Meta refuses for quota (from a simulated Meta 429).
355+
const QUOTA_MESSAGE =
356+
"API error 429: Subscription quota exhausted. Your usage window resets soon. (rate_limit_error)";
357+
// How Muse 1.4.6 ended a turn at a real quota stop (2026-10-13), relayed by a credential proxy.
358+
const RELAYED_QUOTA_MESSAGE =
359+
"API error 429: All credentials for model muse-wonder-1.0-contributor are cooling down (last error: rate_limit_exceeded: Subscription quota exhausted. Your usage window resets at 2026-10-13T10:45:52Z.) (after 10 provider attempts)";
360+
342361
const approval = (turnId: string) => ({
343362
approvalId: "approval-1",
344363
turnId,
@@ -574,6 +593,129 @@ describe("MuseAdapterV2", () => {
574593
}).pipe(Effect.scoped, Effect.provide(testLayer)),
575594
);
576595

596+
it.effect("stops at a Meta usage limit until every exhausted window resets", () =>
597+
Effect.gen(function* () {
598+
const fake = yield* makeFakeMuse();
599+
const harness = yield* makeHarness(fake);
600+
// Meta's report after the last reply that went through: the session window ran out.
601+
yield* fake.emitHostWide("usage/changed", {
602+
...SUBSCRIPTION_USAGE,
603+
window: { ...SUBSCRIPTION_USAGE.window, usedPercent: 103 },
604+
});
605+
const { nativeId } = yield* startConversation(harness, fake);
606+
yield* fake.emit("turn/completed", {
607+
turnId: nativeId,
608+
terminal: "failed",
609+
reason: QUOTA_MESSAGE,
610+
error: { kind: "modelError", message: QUOTA_MESSAGE, retryable: false },
611+
});
612+
const terminal = yield* harness.takeEvent("turn.terminal");
613+
assert.strictEqual(terminal.status, "failed");
614+
assert.strictEqual(terminal.threadDisposition, "reusable");
615+
assert.strictEqual(terminal.failure?.class, "usage_limit");
616+
// The exhausted session window's reset, not the weekly one.
617+
assert.strictEqual(terminal.failure?.resetAt, "2026-10-08T13:19:17.000Z");
618+
assert.isTrue(
619+
harness.allEvents.some(
620+
(event) =>
621+
event.type === "turn_item.updated" &&
622+
event.turnItem.type === "error" &&
623+
event.turnItem.title === "Usage limit reached",
624+
),
625+
);
626+
627+
const next = yield* startConversation(harness, fake, 2);
628+
yield* fake.emit("turn/completed", {
629+
turnId: next.nativeId,
630+
terminal: "failed",
631+
error: { message: "API error 429: Rate limit exceeded. (rate_limit_error)" },
632+
});
633+
const throttled = yield* harness.takeEvent("turn.terminal");
634+
assert.strictEqual(throttled.failure?.class, "provider_error");
635+
}).pipe(Effect.scoped, Effect.provide(testLayer)),
636+
);
637+
638+
it.effect("limits a quota stop without a reset time when no report shows a used-up window", () =>
639+
Effect.gen(function* () {
640+
const fake = yield* makeFakeMuse();
641+
const harness = yield* makeHarness(fake);
642+
// The last report still had room in both windows, so it cannot say when Meta will allow more.
643+
yield* fake.emitHostWide("usage/changed", SUBSCRIPTION_USAGE);
644+
const { nativeId } = yield* startConversation(harness, fake);
645+
yield* fake.emit("turn/completed", {
646+
turnId: nativeId,
647+
terminal: "failed",
648+
error: { message: QUOTA_MESSAGE },
649+
});
650+
const terminal = yield* harness.takeEvent("turn.terminal");
651+
assert.strictEqual(terminal.failure?.class, "usage_limit");
652+
assert.isUndefined(terminal.failure?.resetAt);
653+
}).pipe(Effect.scoped, Effect.provide(testLayer)),
654+
);
655+
656+
it.effect("resumes a quota stop at the reset the refusal names", () =>
657+
Effect.gen(function* () {
658+
const fake = yield* makeFakeMuse();
659+
const harness = yield* makeHarness(fake);
660+
// A fresh host has no report yet; the refusal itself says when the window resets.
661+
const { nativeId } = yield* startConversation(harness, fake);
662+
yield* fake.emit("turn/completed", {
663+
turnId: nativeId,
664+
terminal: "failed",
665+
reason: RELAYED_QUOTA_MESSAGE,
666+
error: { kind: "modelError", message: RELAYED_QUOTA_MESSAGE, retryable: true },
667+
});
668+
const terminal = yield* harness.takeEvent("turn.terminal");
669+
assert.strictEqual(terminal.failure?.class, "usage_limit");
670+
assert.strictEqual(terminal.failure?.resetAt, "2026-10-13T10:45:52.000Z");
671+
}).pipe(Effect.scoped, Effect.provide(testLayer)),
672+
);
673+
674+
it.effect("limits a stop Muse types as quota at the reset it reports", () =>
675+
Effect.gen(function* () {
676+
const fake = yield* makeFakeMuse();
677+
const harness = yield* makeHarness(fake);
678+
const { nativeId } = yield* startConversation(harness, fake);
679+
// The typed quota decides even when the text names no quota.
680+
yield* fake.emit("turn/completed", {
681+
turnId: nativeId,
682+
terminal: "failed",
683+
error: {
684+
kind: "modelError",
685+
message: "Model request refused.",
686+
quota: { limitKind: "tierLimit", resetsAtMs: SUBSCRIPTION_USAGE.window.resetsAtMs },
687+
retryable: true,
688+
},
689+
});
690+
const terminal = yield* harness.takeEvent("turn.terminal");
691+
assert.strictEqual(terminal.failure?.class, "usage_limit");
692+
assert.strictEqual(terminal.failure?.resetAt, "2026-10-08T13:19:17.000Z");
693+
}).pipe(Effect.scoped, Effect.provide(testLayer)),
694+
);
695+
696+
it.effect("skips an unreadable usage report without ending the session", () =>
697+
Effect.gen(function* () {
698+
const fake = yield* makeFakeMuse();
699+
const harness = yield* makeHarness(fake);
700+
yield* fake.emitHostWide("usage/changed", { window: "not usage" });
701+
// A reset no Date can hold would throw once formatted, so that report is skipped too.
702+
yield* fake.emitHostWide("usage/changed", {
703+
...SUBSCRIPTION_USAGE,
704+
window: { ...SUBSCRIPTION_USAGE.window, usedPercent: 103, resetsAtMs: 1e16 },
705+
});
706+
const { nativeId } = yield* startConversation(harness, fake);
707+
yield* fake.emit("turn/completed", {
708+
turnId: nativeId,
709+
terminal: "failed",
710+
error: { message: QUOTA_MESSAGE },
711+
});
712+
const terminal = yield* harness.takeEvent("turn.terminal");
713+
assert.strictEqual(terminal.failure?.class, "usage_limit");
714+
assert.isUndefined(terminal.failure?.resetAt);
715+
assert.strictEqual(fake.closeCount(), 0);
716+
}).pipe(Effect.scoped, Effect.provide(testLayer)),
717+
);
718+
577719
it.effect("moves a resumed session on another model back to the catalog default", () =>
578720
Effect.gen(function* () {
579721
const fake = yield* makeFakeMuse();

‎packages/provider-muse/src/server/adapter.ts‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ import {
5050
MuseDelta,
5151
MuseItemEvent,
5252
MuseSessionResult,
53+
MuseSubscriptionUsage,
5354
MuseTokenUsageEvent,
5455
MuseTodoList,
5556
MuseTurnRetryScheduled,
@@ -61,6 +62,7 @@ import {
6162
museApprovalOptions,
6263
type MuseItem,
6364
} from "./protocol.ts";
65+
import { isMuseUsageLimitFailure, museUsageLimitResetAt } from "./usageLimits.ts";
6466
import {
6567
createMuseSdkHostEffect,
6668
museApprovalMode,
@@ -255,6 +257,7 @@ const INFORMATIONAL_NOTIFICATIONS = new Set([
255257
"session/tokenUsage",
256258
"session/todoListChanged",
257259
"turn/retryScheduled",
260+
"usage/changed",
258261
]);
259262
const TOOLS_WITH_NATIVE_ROWS = new Set(["write_todos", "request_user_input", "workflow"]);
260263
const responseAnswerSchema = Schema.Union([Schema.String, Schema.Array(Schema.String)]);
@@ -325,6 +328,8 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
325328
let thread: OrchestrationV2ProviderThread | undefined;
326329
let active: ActiveTurn | undefined;
327330
let lastProviderTurn: OrchestrationV2ProviderTurn | undefined;
331+
// Meta's newest subscription windows, for when a turn stopped at a usage limit can resume.
332+
let latestSubscriptionUsage: MuseSubscriptionUsage | undefined;
328333
const agentEndedAt = new Map<string, DateTime.Utc>();
329334
const resolveModel = Effect.fnUntraced(function* (selection: ModelSelection) {
330335
if (selection.model !== MUSE_DEFAULT_MODEL) return selection.model;
@@ -934,6 +939,7 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
934939
status: "completed" | "cancelled" | "interrupted" | "failed",
935940
detail?: string,
936941
disposition: "reusable" | "broken" = "reusable",
942+
quota?: { readonly resetsAtMs: number },
937943
) {
938944
if (active !== turn || seenTerminals.has(turn.nativeId)) return;
939945
seenTerminals.add(turn.nativeId);
@@ -969,9 +975,25 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
969975
yield* updateSession(disposition === "broken" ? "error" : "ready", detail ?? null);
970976
active = undefined;
971977
if (status === "failed") {
978+
const usageLimited =
979+
disposition === "reusable" && (quota !== undefined || isMuseUsageLimitFailure(detail));
972980
const failure = makeProviderFailure({
973-
class: disposition === "broken" ? "transport_error" : "provider_error",
981+
class:
982+
disposition === "broken"
983+
? "transport_error"
984+
: usageLimited
985+
? "usage_limit"
986+
: "provider_error",
974987
message: detail ?? "Muse turn failed.",
988+
...(usageLimited
989+
? {
990+
resetAt: museUsageLimitResetAt(
991+
latestSubscriptionUsage,
992+
DateTime.toEpochMillis(completedAt),
993+
{ quota, message: detail },
994+
),
995+
}
996+
: {}),
975997
});
976998
const base = baseItem(turn, `failure:${turn.nativeId}`, completedAt);
977999
yield* emit({
@@ -981,7 +1003,7 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
9811003
...base,
9821004
type: "error",
9831005
status: "failed",
984-
title: null,
1006+
title: usageLimited ? "Usage limit reached" : null,
9851007
completedAt,
9861008
failure,
9871009
},
@@ -1245,6 +1267,11 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
12451267
});
12461268
const handleNotification = Effect.fnUntraced(function* (method: string, data: unknown) {
12471269
const params = yield* decode(recordSchema, data);
1270+
// Account-wide, so it names no session: Meta's subscription windows after a response.
1271+
if (method === "usage/changed") {
1272+
latestSubscriptionUsage = yield* decode(MuseSubscriptionUsage, params);
1273+
return;
1274+
}
12481275
if (params.sessionId !== nativeSessionId) return;
12491276
if (method === "view/gap")
12501277
return yield* protocolError("Muse delivery gap requires session recovery");
@@ -1416,7 +1443,13 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
14161443
? "interrupted"
14171444
: "cancelled"
14181445
: "failed";
1419-
yield* finish(turn, status, result.error?.message ?? result.reason);
1446+
yield* finish(
1447+
turn,
1448+
status,
1449+
result.error?.message ?? result.reason,
1450+
"reusable",
1451+
result.error?.quota,
1452+
);
14201453
break;
14211454
}
14221455
case "approval/requested":

‎packages/provider-muse/src/server/protocol.ts‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,9 +152,39 @@ export const MuseTurnCompleted = Schema.Struct({
152152
turnId: NonEmptyString,
153153
terminal: NonEmptyString,
154154
reason: Schema.optional(Schema.String),
155-
error: Schema.optional(Schema.Struct({ message: Schema.String })),
155+
error: Schema.optional(
156+
Schema.Struct({
157+
message: Schema.String,
158+
// Present when Muse recognized the stop as a used-up quota.
159+
quota: Schema.optional(Schema.Struct({ resetsAtMs: Schema.Finite })),
160+
}),
161+
),
156162
usage: Schema.optional(MuseUsage),
157163
});
164+
/** The latest instant a JavaScript Date holds; formatting a later one throws. */
165+
export const LATEST_DATE_MS = 8.64e15;
166+
const DateMillis = Schema.Finite.check(
167+
Schema.isGreaterThan(0),
168+
Schema.isLessThanOrEqualTo(LATEST_DATE_MS),
169+
);
170+
/**
171+
* `usage/changed` params: the Meta subscription windows the latest model
172+
* response reported. Percentages are Meta's, verbatim, and may exceed 100.
173+
*/
174+
export const MuseSubscriptionUsage = Schema.Struct({
175+
observedAtMs: DateMillis,
176+
tier: Schema.String,
177+
window: Schema.Struct({
178+
usedPercent: Schema.Finite.check(Schema.isGreaterThanOrEqualTo(0)),
179+
windowDurationMins: Schema.Finite.check(Schema.isGreaterThan(0)),
180+
resetsAtMs: DateMillis,
181+
}),
182+
weekly: Schema.Struct({
183+
usedPercent: Schema.Finite.check(Schema.isGreaterThanOrEqualTo(0)),
184+
resetsAtMs: DateMillis,
185+
}),
186+
});
187+
export type MuseSubscriptionUsage = typeof MuseSubscriptionUsage.Type;
158188
export const MuseTurnRetryScheduled = Schema.Struct({
159189
turnId: NonEmptyString,
160190
nextAttempt: Schema.Int.check(Schema.isGreaterThan(0)),
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
import { describe, expect, it } from "@effect/vitest";
2+
3+
import type { MuseSubscriptionUsage } from "./protocol.ts";
4+
import { isMuseUsageLimitFailure, museUsageLimitResetAt } from "./usageLimits.ts";
5+
6+
// What Meta reported after a reply on 2026-10-08, as Muse forwards it.
7+
const usage = {
8+
observedAtMs: 1791447762610,
9+
tier: "tier-1",
10+
window: { usedPercent: 0, windowDurationMins: 300, resetsAtMs: 1791465557000 },
11+
weekly: { usedPercent: 5, resetsAtMs: 1791763200000 },
12+
} satisfies MuseSubscriptionUsage;
13+
const nowMs = usage.observedAtMs;
14+
// How Muse 1.4.6 ended a turn at a real quota stop (2026-10-13), relayed by a credential proxy.
15+
const realRefusal =
16+
"API error 429: All credentials for model muse-wonder-1.0-contributor are cooling down (last error: rate_limit_exceeded: Subscription quota exhausted. Your usage window resets at 2026-10-13T10:45:52Z.) (after 10 provider attempts)";
17+
18+
describe("Muse usage limit failures", () => {
19+
it("recognizes Meta's quota refusal and no other failure", () => {
20+
expect(
21+
isMuseUsageLimitFailure(
22+
"API error 429: Subscription quota exhausted. Your usage window resets soon. (rate_limit_error)",
23+
),
24+
).toBe(true);
25+
expect(isMuseUsageLimitFailure(realRefusal)).toBe(true);
26+
// A plain rate limit retries on its own; it is not a used-up subscription window.
27+
expect(isMuseUsageLimitFailure("API error 429: Rate limit exceeded. (rate_limit_error)")).toBe(
28+
false,
29+
);
30+
expect(isMuseUsageLimitFailure("Subscription quota exhausted.")).toBe(false);
31+
expect(isMuseUsageLimitFailure(undefined)).toBe(false);
32+
});
33+
34+
it("resumes once every exhausted window has reset", () => {
35+
const sessionOut = { ...usage, window: { ...usage.window, usedPercent: 103 } };
36+
expect(museUsageLimitResetAt(sessionOut, nowMs)).toBe("2026-10-08T13:19:17.000Z");
37+
const bothOut = { ...sessionOut, weekly: { ...usage.weekly, usedPercent: 100 } };
38+
expect(museUsageLimitResetAt(bothOut, nowMs)).toBe("2026-10-12T00:00:00.000Z");
39+
});
40+
41+
it("takes the reset a refusal names or types when no report has one", () => {
42+
expect(museUsageLimitResetAt(undefined, nowMs, { message: realRefusal })).toBe(
43+
"2026-10-13T10:45:52.000Z",
44+
);
45+
const quota = { resetsAtMs: usage.window.resetsAtMs };
46+
expect(museUsageLimitResetAt(usage, nowMs, { quota })).toBe("2026-10-08T13:19:17.000Z");
47+
// A used-up weekly window still holds the turn back after the typed reset.
48+
const weeklyOut = { ...usage, weekly: { ...usage.weekly, usedPercent: 100 } };
49+
expect(museUsageLimitResetAt(weeklyOut, nowMs, { quota })).toBe("2026-10-12T00:00:00.000Z");
50+
});
51+
52+
it("has no reset time unless a report shows a used-up window that resets later", () => {
53+
expect(museUsageLimitResetAt(undefined, nowMs)).toBeNull();
54+
expect(museUsageLimitResetAt(usage, nowMs)).toBeNull();
55+
const sessionOut = { ...usage, window: { ...usage.window, usedPercent: 100 } };
56+
expect(museUsageLimitResetAt(sessionOut, usage.window.resetsAtMs)).toBeNull();
57+
const afterNamedReset = Date.parse("2026-10-14T00:00:00Z");
58+
expect(museUsageLimitResetAt(undefined, afterNamedReset, { message: realRefusal })).toBeNull();
59+
});
60+
});

0 commit comments

Comments
 (0)