Fix failover for zhipuai 1310 Weekly/Monthly Limit Exhausted (openclaw#33813)

zhouhe-xydt · altaywtf · web-flow · commit a65d70f84b4e · 2026-03-06T12:04:09.000+03:00
Merged via squash. Prepared head SHA: 3dc441e Co-authored-by: zhouhe-xydt <265407618+zhouhe-xydt@users.noreply.github.com> Co-authored-by: altaywtf <9790196+altaywtf@users.noreply.github.com> Reviewed-by: @altaywtf
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -194,6 +194,7 @@ Docs: https://docs.openclaw.ai
 - Memory/flush default prompt: ban timestamped variant filenames during default memory flush runs so durable notes stay in the canonical daily `memory/YYYY-MM-DD.md` file. (#34951) thanks @zerone0x.
 - Agents/reply delivery timing: flush embedded Pi block replies before waiting on compaction retries so already-generated assistant replies reach channels before compaction wait completes. (#35489) thanks @Sid-Qin.
 - Agents/gateway config guidance: stop exposing `config.schema` through the agent `gateway` tool, remove prompt/docs guidance that told agents to call it, and keep agents on `config.get` plus `config.patch`/`config.apply` for config changes. (#7382) thanks @kakuteki.
+- Agents/failover: classify periodic provider limit exhaustion text (for example `Weekly/Monthly Limit Exhausted`) as `rate_limit` while keeping explicit `402 Payment Required` variants in billing, so failover continues without misclassifying billing-wrapped quota errors. (#33813) thanks @zhouhe-xydt.
 
 ## 2026.3.2
 
diff --git a/src/agents/failover-error.test.ts b/src/agents/failover-error.test.ts
@@ -22,6 +22,10 @@ const OPENROUTER_CREDITS_MESSAGE = "Payment Required: insufficient credits";
 // https://github.com/openclaw/openclaw/issues/23440
 const INSUFFICIENT_QUOTA_PAYLOAD =
   '{"type":"error","error":{"type":"insufficient_quota","message":"Your account has insufficient quota balance to run this request."}}';
+// Issue-backed ZhipuAI/GLM quota-exhausted log from #33785:
+// https://github.com/openclaw/openclaw/issues/33785
+const ZHIPUAI_WEEKLY_MONTHLY_LIMIT_EXHAUSTED_MESSAGE =
+  "LLM error 1310: Weekly/Monthly Limit Exhausted. Your limit will reset at 2026-03-06 22:19:54 (request_id: 20260303141547610b7f574d1b44cb)";
 // AWS Bedrock 429 ThrottlingException / 503 ServiceUnavailable:
 // https://docs.aws.amazon.com/bedrock/latest/userguide/troubleshooting-api-error-codes.html
 const BEDROCK_THROTTLING_EXCEPTION_MESSAGE =
@@ -113,6 +117,27 @@ describe("failover-error", () => {
     ).toBe("billing");
   });
 
+  it("treats zhipuai weekly/monthly limit exhausted as rate_limit", () => {
+    expect(
+      resolveFailoverReasonFromError({
+        message: ZHIPUAI_WEEKLY_MONTHLY_LIMIT_EXHAUSTED_MESSAGE,
+      }),
+    ).toBe("rate_limit");
+    expect(
+      resolveFailoverReasonFromError({
+        message: "LLM error: monthly limit reached",
+      }),
+    ).toBe("rate_limit");
+  });
+
+  it("keeps raw-text 402 weekly/monthly limit errors in billing", () => {
+    expect(
+      resolveFailoverReasonFromError({
+        message: "402 Payment Required: Weekly/Monthly Limit Exhausted",
+      }),
+    ).toBe("billing");
+  });
+
   it("infers format errors from error messages", () => {
     expect(
       resolveFailoverReasonFromError({
diff --git a/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts b/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts
@@ -535,6 +535,14 @@ describe("classifyFailoverReason", () => {
     ).toBe("rate_limit");
     expect(classifyFailoverReason("all credentials for model x are cooling down")).toBeNull();
     expect(classifyFailoverReason("invalid request format")).toBe("format");
+    expect(classifyFailoverReason("credit balance too low")).toBe("billing");
+    // Billing with "limit exhausted" must stay billing, not rate_limit (avoids key-disable regression)
+    expect(
+      classifyFailoverReason("HTTP 402 payment required. Your limit exhausted for this plan."),
+    ).toBe("billing");
+    expect(classifyFailoverReason("402 Payment Required: Weekly/Monthly Limit Exhausted")).toBe(
+      "billing",
+    );
     expect(classifyFailoverReason(INSUFFICIENT_QUOTA_PAYLOAD)).toBe("billing");
     expect(classifyFailoverReason("deadline exceeded")).toBe("timeout");
     expect(classifyFailoverReason("request ended without sending any chunks")).toBe("timeout");
@@ -584,6 +592,17 @@ describe("classifyFailoverReason", () => {
     // but it should not be treated as provider overload / rate limit.
     expect(classifyFailoverReason("LLM error: service unavailable")).toBe("timeout");
   });
+  it("classifies zhipuai Weekly/Monthly Limit Exhausted as rate_limit (#33785)", () => {
+    expect(
+      classifyFailoverReason(
+        "LLM error 1310: Weekly/Monthly Limit Exhausted. Your limit will reset at 2026-03-06 22:19:54 (request_id: 20260303141547610b7f574d1b44cb)",
+      ),
+    ).toBe("rate_limit");
+    // Independent coverage for broader periodic limit patterns.
+    expect(classifyFailoverReason("LLM error: weekly/monthly limit reached")).toBe("rate_limit");
+    expect(classifyFailoverReason("LLM error: monthly limit reached")).toBe("rate_limit");
+    expect(classifyFailoverReason("LLM error: daily limit exceeded")).toBe("rate_limit");
+  });
   it("classifies permanent auth errors as auth_permanent", () => {
     expect(classifyFailoverReason("invalid_api_key")).toBe("auth_permanent");
     expect(classifyFailoverReason("Your api key has been revoked")).toBe("auth_permanent");
diff --git a/src/agents/pi-embedded-helpers/errors.ts b/src/agents/pi-embedded-helpers/errors.ts
@@ -8,6 +8,7 @@ import {
   isAuthPermanentErrorMessage,
   isBillingErrorMessage,
   isOverloadedErrorMessage,
+  isPeriodicUsageLimitErrorMessage,
   isRateLimitErrorMessage,
   isTimeoutErrorMessage,
   matchesFormatErrorPattern,
@@ -842,6 +843,9 @@ export function classifyFailoverReason(raw: string): FailoverReason | null {
   if (isJsonApiInternalServerError(raw)) {
     return "timeout";
   }
+  if (isPeriodicUsageLimitErrorMessage(raw)) {
+    return isBillingErrorMessage(raw) ? "billing" : "rate_limit";
+  }
   if (isRateLimitErrorMessage(raw)) {
     return "rate_limit";
   }
diff --git a/src/agents/pi-embedded-helpers/failover-matches.ts b/src/agents/pi-embedded-helpers/failover-matches.ts
@@ -1,5 +1,8 @@
 type ErrorPattern = RegExp | string;
 
+const PERIODIC_USAGE_LIMIT_RE =
+  /\b(?:daily|weekly|monthly)(?:\/(?:daily|weekly|monthly))* (?:usage )?limit(?:s)?(?: (?:exhausted|reached|exceeded))?\b/i;
+
 const ERROR_PATTERNS = {
   rateLimit: [
     /rate[_ ]limit|too many requests|429/,
@@ -117,6 +120,10 @@ export function isTimeoutErrorMessage(raw: string): boolean {
   return matchesErrorPatterns(raw, ERROR_PATTERNS.timeout);
 }
 
+export function isPeriodicUsageLimitErrorMessage(raw: string): boolean {
+  return PERIODIC_USAGE_LIMIT_RE.test(raw);
+}
+
 export function isBillingErrorMessage(raw: string): boolean {
   const value = raw.toLowerCase();
   if (!value) {
