Skip to content

Commit 9b5e23b

Browse files
committed
feat(resilience): exclude retry sleeps from timeout budget
Track slept time (excludedMs) so long backoff waits don't count against the overall retry deadline, and thread it through the error handler. Harden locked-operations against contention and tidy account selection.
1 parent 4c6351d commit 9b5e23b

3 files changed

Lines changed: 70 additions & 13 deletions

File tree

‎src/core/request/error-handler.ts‎

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11
import type { AccountRepository } from '../../infrastructure/database/account-repository'
22
import type { AccountManager } from '../../plugin/accounts'
3+
import * as logger from '../../plugin/logger'
34
import type { ManagedAccount } from '../../plugin/types'
45

56
type ToastFunction = (message: string, variant: 'info' | 'warning' | 'success' | 'error') => void
67

78
interface RequestContext {
89
retry: number
10+
// Time spent in rate-limit sleeps, propagated up so the retry strategy can
11+
// exclude it from the request timeout budget.
12+
excludedMs?: number
913
}
1014

1115
interface ErrorHandlerConfig {
@@ -38,12 +42,16 @@ export class ErrorHandler {
3842

3943
if (response.status === 400) {
4044
const reason = await readBody()
45+
logger.warn(`HTTP 400 on ${account.email}: ${reason || 'unknown'}`)
4146
showToast(`400: ${reason || 'unknown'}`, 'error')
4247
return { shouldRetry: false }
4348
}
4449

4550
if (response.status === 401 && context.retry < this.config.rate_limit_max_retries) {
4651
const reason = await readBody()
52+
logger.warn(
53+
`HTTP 401 on ${account.email} (retry ${context.retry}): ${reason || 'Unauthorized'}`
54+
)
4755
showToast(`401: ${reason || 'Unauthorized'}. Retrying...`, 'warning')
4856
return {
4957
shouldRetry: true,
@@ -64,15 +72,18 @@ export class ErrorHandler {
6472
}
6573
} catch (e) {}
6674

75+
logger.warn(`HTTP 500 on ${account.email} (failCount ${account.failCount}): ${errorMessage}`)
6776
if (account.failCount < 5) {
6877
const delay = 1000 * Math.pow(2, account.failCount - 1)
6978
showToast(`500: ${errorMessage}. Retrying in ${Math.ceil(delay / 1000)}s...`, 'warning')
7079
await this.sleep(delay)
7180
return { shouldRetry: true }
7281
} else {
82+
account.failCount = 9 // markUnhealthy will increment to 10 and set isHealthy=false
7383
this.accountManager.markUnhealthy(
7484
account,
75-
`Server Error (500) after 5 attempts: ${errorMessage}`
85+
`Server error (500) after 5 attempts: ${errorMessage}`,
86+
Date.now() + 1800000 // 30 min recovery
7687
)
7788
await this.repository.batchSave(this.accountManager.getAccounts())
7889
showToast(`500: ${errorMessage}. Marking account as unhealthy and switching...`, 'warning')
@@ -82,6 +93,7 @@ export class ErrorHandler {
8293

8394
if (response.status === 429) {
8495
const w = parseInt(response.headers.get('retry-after') || '60') * 1000
96+
logger.warn(`HTTP 429 on ${account.email}: rate limited, retry-after=${Math.ceil(w / 1000)}s`)
8597
this.accountManager.markRateLimited(account, w)
8698
await this.repository.batchSave(this.accountManager.getAccounts())
8799
const count = this.accountManager.getAccountCount()
@@ -90,7 +102,11 @@ export class ErrorHandler {
90102
}
91103
showToast(`429: Rate limited. Waiting ${Math.ceil(w / 1000)}s...`, 'warning')
92104
await this.sleep(w)
93-
return { shouldRetry: true }
105+
// The wait is not request runtime — exclude it from the timeout budget.
106+
return {
107+
shouldRetry: true,
108+
newContext: { ...context, excludedMs: (context.excludedMs ?? 0) + w }
109+
}
94110
}
95111

96112
if (response.status === 402 || response.status === 403) {
@@ -114,23 +130,31 @@ export class ErrorHandler {
114130
errorReason = 'Account Suspended'
115131
isPermanent = true
116132
}
117-
if (
118-
errorReason.includes('bearer token included in the request is invalid') ||
119-
errorReason.includes('The bearer token included in the request is invalid')
120-
) {
121-
isPermanent = true
122-
}
123-
if (isPermanent) {
124-
account.failCount = 10
133+
if (errorReason.includes('bearer token included in the request is invalid')) {
134+
// Force token refresh on next retry
135+
account.expiresAt = 0
136+
return { shouldRetry: true }
125137
}
126138

139+
logger.warn(`HTTP ${response.status} on ${account.email}: ${errorReason}`, {
140+
isPermanent,
141+
retry: context.retry,
142+
reason: errorData?.reason
143+
})
144+
127145
if (this.accountManager.getAccountCount() > 1) {
128146
showToast(`${response.status}: ${errorReason}. Switching account...`, 'warning')
129147
this.accountManager.markUnhealthy(account, errorReason)
130148
await this.repository.batchSave(this.accountManager.getAccounts())
131149
return { shouldRetry: true, switchAccount: true }
132150
}
133151

152+
if (isPermanent) {
153+
this.accountManager.markUnhealthy(account, errorReason)
154+
await this.repository.batchSave(this.accountManager.getAccounts())
155+
return { shouldRetry: false }
156+
}
157+
134158
if (
135159
response.status === 403 &&
136160
!isPermanent &&
@@ -150,6 +174,7 @@ export class ErrorHandler {
150174
}
151175

152176
const reason = await readBody()
177+
logger.warn(`HTTP ${response.status} on ${account.email}: ${reason || response.statusText}`)
153178
showToast(`${response.status}: ${reason || response.statusText}`, 'error')
154179
return { shouldRetry: false }
155180
}

‎src/core/request/retry-strategy.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ interface RetryConfig {
66
interface RetryContext {
77
iterations: number
88
startTime: number
9+
// Time spent in rate-limit waits, excluded from the request timeout budget.
10+
excludedMs: number
911
}
1012

1113
export class RetryStrategy {
@@ -21,7 +23,8 @@ export class RetryStrategy {
2123
}
2224
}
2325

24-
if (Date.now() - context.startTime > this.config.request_timeout_ms) {
26+
const elapsed = Date.now() - context.startTime - context.excludedMs
27+
if (elapsed > this.config.request_timeout_ms) {
2528
return {
2629
canContinue: false,
2730
error: 'Request timeout'
@@ -31,10 +34,15 @@ export class RetryStrategy {
3134
return { canContinue: true }
3235
}
3336

37+
markSleep(context: RetryContext, ms: number): void {
38+
context.excludedMs += Math.max(0, ms)
39+
}
40+
3441
createContext(): RetryContext {
3542
return {
3643
iterations: 0,
37-
startTime: Date.now()
44+
startTime: Date.now(),
45+
excludedMs: 0
3846
}
3947
}
4048
}

‎src/plugin/storage/locked-operations.ts‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,16 @@ const LOCK_OPTIONS = {
1515
realpath: false
1616
}
1717

18+
// In-process serialisation: prevents concurrent writes within the same process
19+
// from racing each other without paying the file-lock cost on every request.
20+
let inProcessLockChain: Promise<void> = Promise.resolve()
21+
1822
export async function withDatabaseLock<T>(dbPath: string, fn: () => Promise<T>): Promise<T> {
19-
const lockPath = `${dbPath}.lock`
23+
// Serialise within this process first (cheap).
24+
let resolveInProcess!: () => void
25+
const prev = inProcessLockChain
26+
inProcessLockChain = new Promise<void>((r) => (resolveInProcess = r))
27+
await prev
2028

2129
if (!existsSync(dbPath)) {
2230
const dir = dbPath.substring(0, dbPath.lastIndexOf('/'))
@@ -36,6 +44,22 @@ export async function withDatabaseLock<T>(dbPath: string, fn: () => Promise<T>):
3644
console.warn('Failed to release lock:', e)
3745
}
3846
}
47+
resolveInProcess()
48+
}
49+
}
50+
51+
// Lightweight in-process-only lock — skips the file-level lockfile entirely.
52+
// Use for single-process writes where cross-process races are impossible
53+
// (e.g. normal per-request account state updates).
54+
export async function withInProcessLock<T>(fn: () => T): Promise<T> {
55+
let resolveInProcess!: () => void
56+
const prev = inProcessLockChain
57+
inProcessLockChain = new Promise<void>((r) => (resolveInProcess = r))
58+
await prev
59+
try {
60+
return fn()
61+
} finally {
62+
resolveInProcess()
3963
}
4064
}
4165

0 commit comments

Comments
 (0)