Skip to content

Commit a6ccb5f

Browse files
steipetesteipete
committed
fix(discord): retry transient outbound failures
1 parent d961235 commit a6ccb5f

6 files changed

Lines changed: 184 additions & 6 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ Docs: https://docs.openclaw.ai
4040
- Discord/doctor: migrate unsupported per-channel `agentId` entries under guild channel config into top-level `bindings[]` routes, so `openclaw doctor --fix` preserves the intended agent route instead of stripping it as an unknown key. Fixes #62455. Thanks @lobster-biscuit.
4141
- Discord/DMs: set inbound direct-message `ctx.To` to the semantic `user:<id>` target while keeping delivery routed through the DM channel, so mirror and recovery paths do not treat DMs as channel conversations. Fixes #68126. Thanks @illuminate0623.
4242
- Discord/DMs: keep no-guild inbound messages on direct-message routing when Discord channel lookup is temporarily unavailable, preventing degraded DMs from forking into channel sessions. Fixes #59817. Thanks @DooPeePey.
43+
- Discord: retry outbound API calls on HTTP 5xx, request-timeout, and transient transport failures instead of only Discord rate limits, reducing dropped cron and agent replies during short Discord or network outages. Fixes #52396. Thanks @sunshineo.
4344
- Gateway/config: log config health-state write failures instead of silently hiding config observe-recovery write errors. Thanks @sallyom.
4445
- Diagnostics: reset stuck-session timers on reply, tool, status, block, and ACP progress events, and back off repeated `session.stuck` diagnostics while a session remains unchanged. Supersedes #72010. Thanks @rubencu.
4546

‎docs/concepts/retry.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,9 @@ title: "Retry policy"
3737

3838
### Discord
3939

40-
- Retries only on rate-limit errors (HTTP 429).
40+
- Retries on rate-limit errors (HTTP 429), request timeouts, HTTP 5xx responses,
41+
and transient transport failures such as DNS lookup failures, connection
42+
resets, socket closes, and fetch failures.
4143
- Uses Discord `retry_after` when available, otherwise exponential backoff.
4244

4345
### Telegram

‎extensions/discord/src/delivery-retry.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import {
55
type RetryConfig,
66
} from "openclaw/plugin-sdk/retry-runtime";
77
import { resolveDiscordAccount } from "./accounts.js";
8+
import { DiscordError } from "./internal/discord.js";
89

910
const DISCORD_DELIVERY_RETRY_DEFAULTS = {
1011
attempts: 3,
@@ -13,7 +14,10 @@ const DISCORD_DELIVERY_RETRY_DEFAULTS = {
1314
jitter: 0,
1415
} satisfies Required<RetryConfig>;
1516

16-
function isRetryableDiscordDeliveryError(err: unknown): boolean {
17+
export function isRetryableDiscordDeliveryError(err: unknown): boolean {
18+
if (err instanceof DiscordError) {
19+
return false;
20+
}
1721
const status = (err as { status?: number }).status ?? (err as { statusCode?: number }).statusCode;
1822
return status === 429 || (status !== undefined && status >= 500);
1923
}

‎extensions/discord/src/retry.test.ts‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
import { describe, expect, it, vi } from "vitest";
2+
import { isRetryableDiscordDeliveryError } from "./delivery-retry.js";
3+
import { DiscordError, RateLimitError } from "./internal/discord.js";
4+
import { createDiscordRetryRunner, isRetryableDiscordTransientError } from "./retry.js";
5+
6+
const ZERO_DELAY_RETRY = { attempts: 2, minDelayMs: 0, maxDelayMs: 0, jitter: 0 };
7+
8+
function createRateLimitError(retryAfter = 0): RateLimitError {
9+
const response = new Response(null, {
10+
status: 429,
11+
headers: {
12+
"X-RateLimit-Scope": "user",
13+
"X-RateLimit-Bucket": "bucket-1",
14+
},
15+
});
16+
const RateLimitErrorCtor = RateLimitError as unknown as new (
17+
response: Response,
18+
body: { message: string; retry_after: number; global: boolean },
19+
) => RateLimitError;
20+
return new RateLimitErrorCtor(response, {
21+
message: "rate limited",
22+
retry_after: retryAfter,
23+
global: false,
24+
});
25+
}
26+
27+
describe("isRetryableDiscordTransientError", () => {
28+
it.each([
29+
["rate limit", createRateLimitError()],
30+
["408 status", Object.assign(new Error("request timeout"), { status: 408 })],
31+
["502 status", Object.assign(new Error("bad gateway"), { status: 502 })],
32+
["503 statusCode", Object.assign(new Error("service unavailable"), { statusCode: 503 })],
33+
["fetch failed", new TypeError("fetch failed")],
34+
["ECONNRESET", Object.assign(new Error("socket hang up"), { code: "ECONNRESET" })],
35+
["ETIMEDOUT cause", new Error("request failed", { cause: { code: "ETIMEDOUT" } })],
36+
["abort", Object.assign(new Error("aborted"), { name: "AbortError" })],
37+
])("retries %s", (_name, err) => {
38+
expect(isRetryableDiscordTransientError(err)).toBe(true);
39+
});
40+
41+
it.each([
42+
["400 status", Object.assign(new Error("bad request"), { status: 400 })],
43+
["403 status", Object.assign(new Error("missing permissions"), { statusCode: 403 })],
44+
["unknown channel", new Error("Unknown Channel")],
45+
["plain string", "fetch failed"],
46+
])("does not retry %s", (_name, err) => {
47+
expect(isRetryableDiscordTransientError(err)).toBe(false);
48+
});
49+
});
50+
51+
describe("createDiscordRetryRunner", () => {
52+
it("retries transient transport errors", async () => {
53+
const fn = vi.fn().mockRejectedValueOnce(new TypeError("fetch failed")).mockResolvedValue("ok");
54+
const runner = createDiscordRetryRunner({ retry: ZERO_DELAY_RETRY });
55+
56+
await expect(runner(fn, "send")).resolves.toBe("ok");
57+
expect(fn).toHaveBeenCalledTimes(2);
58+
});
59+
60+
it("stops after configured transient retry attempts", async () => {
61+
const fn = vi.fn().mockRejectedValue(new TypeError("fetch failed"));
62+
const runner = createDiscordRetryRunner({ retry: ZERO_DELAY_RETRY });
63+
64+
await expect(runner(fn, "send")).rejects.toThrow("fetch failed");
65+
expect(fn).toHaveBeenCalledTimes(2);
66+
});
67+
});
68+
69+
describe("isRetryableDiscordDeliveryError", () => {
70+
it("retries status-coded errors from injected delivery dependencies", () => {
71+
expect(
72+
isRetryableDiscordDeliveryError(Object.assign(new Error("bad gateway"), { status: 502 })),
73+
).toBe(true);
74+
});
75+
76+
it("does not retry Discord client errors after the request runner handled them", () => {
77+
const err = new DiscordError(new Response("upstream", { status: 502 }), {
78+
message: "Bad Gateway",
79+
});
80+
81+
expect(isRetryableDiscordDeliveryError(err)).toBe(false);
82+
});
83+
});

‎extensions/discord/src/retry.ts‎

Lines changed: 72 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
import {
2+
collectErrorGraphCandidates,
3+
extractErrorCode,
4+
formatErrorMessage,
5+
readErrorName,
6+
} from "openclaw/plugin-sdk/error-runtime";
17
import {
28
createRateLimitRetryRunner,
39
type RetryConfig,
@@ -12,6 +18,71 @@ const DISCORD_RETRY_DEFAULTS = {
1218
jitter: 0.1,
1319
} satisfies RetryConfig;
1420

21+
const DISCORD_RETRYABLE_STATUS_CODES = new Set([408, 429]);
22+
const DISCORD_RETRYABLE_ERROR_CODES = new Set([
23+
"EAI_AGAIN",
24+
"ECONNREFUSED",
25+
"ECONNRESET",
26+
"ENETUNREACH",
27+
"ENOTFOUND",
28+
"EPIPE",
29+
"ETIMEDOUT",
30+
"UND_ERR_BODY_TIMEOUT",
31+
"UND_ERR_CONNECT_TIMEOUT",
32+
"UND_ERR_HEADERS_TIMEOUT",
33+
"UND_ERR_SOCKET",
34+
]);
35+
const DISCORD_TRANSIENT_MESSAGE_RE =
36+
/\b(?:bad gateway|fetch failed|network error|networkerror|service unavailable|socket hang up|temporarily unavailable|timed out|timeout)\b|connection (?:closed|reset|refused)/i;
37+
38+
function readDiscordErrorStatus(err: unknown): number | undefined {
39+
if (!err || typeof err !== "object") {
40+
return undefined;
41+
}
42+
const raw =
43+
"status" in err && err.status !== undefined
44+
? err.status
45+
: "statusCode" in err && err.statusCode !== undefined
46+
? err.statusCode
47+
: undefined;
48+
if (typeof raw === "number" && Number.isFinite(raw)) {
49+
return raw;
50+
}
51+
if (typeof raw === "string" && /^\d+$/.test(raw)) {
52+
return Number(raw);
53+
}
54+
return undefined;
55+
}
56+
57+
export function isRetryableDiscordTransientError(err: unknown): boolean {
58+
if (err instanceof RateLimitError) {
59+
return true;
60+
}
61+
for (const candidate of collectErrorGraphCandidates(err, (current) => [
62+
current.cause,
63+
current.error,
64+
])) {
65+
const status = readDiscordErrorStatus(candidate);
66+
if (status !== undefined && (DISCORD_RETRYABLE_STATUS_CODES.has(status) || status >= 500)) {
67+
return true;
68+
}
69+
const code = extractErrorCode(candidate);
70+
if (code && DISCORD_RETRYABLE_ERROR_CODES.has(code.toUpperCase())) {
71+
return true;
72+
}
73+
if (readErrorName(candidate) === "AbortError") {
74+
return true;
75+
}
76+
if (
77+
(candidate instanceof Error || (candidate !== null && typeof candidate === "object")) &&
78+
DISCORD_TRANSIENT_MESSAGE_RE.test(formatErrorMessage(candidate))
79+
) {
80+
return true;
81+
}
82+
}
83+
return false;
84+
}
85+
1586
export function createDiscordRetryRunner(params: {
1687
retry?: RetryConfig;
1788
configRetry?: RetryConfig;
@@ -21,7 +92,7 @@ export function createDiscordRetryRunner(params: {
2192
...params,
2293
defaults: DISCORD_RETRY_DEFAULTS,
2394
logLabel: "discord",
24-
shouldRetry: (err) => err instanceof RateLimitError,
95+
shouldRetry: isRetryableDiscordTransientError,
2596
retryAfterMs: (err) => (err instanceof RateLimitError ? err.retryAfter * 1000 : undefined),
2697
});
2798
}

‎extensions/discord/src/send.creates-thread.test.ts‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -547,16 +547,33 @@ describe("retry rate limits", () => {
547547
expect(postMock).toHaveBeenCalledTimes(2);
548548
});
549549

550-
it("does not retry non-rate-limit errors", async () => {
550+
it("does not retry permanent non-rate-limit errors", async () => {
551551
const { rest, postMock } = makeDiscordRest();
552-
postMock.mockRejectedValueOnce(new Error("network error"));
552+
postMock.mockRejectedValueOnce(new Error("invalid request"));
553553

554554
await expect(
555555
sendMessageDiscord("channel:789", "hello", discordClientOpts(rest)),
556-
).rejects.toThrow("network error");
556+
).rejects.toThrow("invalid request");
557557
expect(postMock).toHaveBeenCalledTimes(1);
558558
});
559559

560+
it("retries transient network errors", async () => {
561+
const { rest, postMock } = makeDiscordRest();
562+
postMock
563+
.mockRejectedValueOnce(new TypeError("fetch failed"))
564+
.mockResolvedValueOnce({ id: "msg1", channel_id: "789" });
565+
566+
const result = await sendMessageDiscord("channel:789", "hello", {
567+
cfg: DISCORD_TEST_CFG,
568+
rest,
569+
token: "t",
570+
retry: { attempts: 2, minDelayMs: 0, maxDelayMs: 0, jitter: 0 },
571+
});
572+
573+
expect(result).toEqual({ messageId: "msg1", channelId: "789" });
574+
expect(postMock).toHaveBeenCalledTimes(2);
575+
});
576+
560577
it("retries reactions on rate limits", async () => {
561578
const { rest, putMock } = makeDiscordRest();
562579
const rateLimitError = createMockRateLimitError(0);

0 commit comments

Comments
 (0)