Skip to content

Commit 288893b

Browse files
committed
fix(usage): throttle failed pricing refreshes
1 parent ab645b6 commit 288893b

2 files changed

Lines changed: 85 additions & 1 deletion

File tree

‎apps/server/src/usage/UsageService.test.ts‎

Lines changed: 80 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1850,7 +1850,7 @@ describe("UsageService", () => {
18501850
assert.strictEqual(ratesFetches, 2);
18511851

18521852
yield* service.readSummary(WINDOW);
1853-
assert.strictEqual(ratesFetches, 4);
1853+
assert.strictEqual(ratesFetches, 2);
18541854
}).pipe(Effect.scoped),
18551855
);
18561856

@@ -1929,6 +1929,7 @@ describe("UsageService", () => {
19291929
assert.deepStrictEqual(cached.buckets, first.buckets);
19301930

19311931
status = 200;
1932+
yield* TestClock.adjust(Duration.minutes(1));
19321933
const recovered = yield* service.refreshRates;
19331934
assert.strictEqual(recovered.status, "fresh");
19341935
assert.notStrictEqual(recovered.fetchedAt, first.pricing.fetchedAt);
@@ -2036,6 +2037,84 @@ describe("UsageService", () => {
20362037
}).pipe(Effect.scoped),
20372038
);
20382039

2040+
it.live.each([false, true])(
2041+
"throttles failed pricing loads from completion with cached rates: %s",
2042+
(hasCachedRates) =>
2043+
Effect.gen(function* () {
2044+
const { settings, home } = yield* setup;
2045+
const primaryStarted = yield* Deferred.make<void>();
2046+
const fallbackStarted = yield* Deferred.make<void>();
2047+
let responding = true;
2048+
let requests = 0;
2049+
const service = yield* UsageService.make.pipe(
2050+
Effect.provide(
2051+
layerService({
2052+
prefix: "usage-service-rates-retry-floor-test",
2053+
home,
2054+
settings,
2055+
onRatesFetch: () => {
2056+
requests += 1;
2057+
},
2058+
ratesResponse: (request) =>
2059+
responding
2060+
? Effect.succeed(
2061+
Response.json({
2062+
"claude-fable-5": {
2063+
input_cost_per_token: 1e-5,
2064+
output_cost_per_token: 5e-5,
2065+
},
2066+
}),
2067+
)
2068+
: Deferred.succeed(
2069+
request.url.startsWith("https://api.github.com/")
2070+
? fallbackStarted
2071+
: primaryStarted,
2072+
undefined,
2073+
).pipe(Effect.andThen(Effect.never)),
2074+
}),
2075+
),
2076+
);
2077+
let fetchedAt: string | null = null;
2078+
if (hasCachedRates) {
2079+
fetchedAt = (yield* service.refreshRates).fetchedAt;
2080+
yield* TestClock.adjust(Duration.hours(24));
2081+
}
2082+
responding = false;
2083+
const refresh = yield* service.refreshRates.pipe(Effect.forkChild);
2084+
yield* Deferred.await(primaryStarted);
2085+
yield* TestClock.adjust(Duration.seconds(10));
2086+
yield* Deferred.await(fallbackStarted);
2087+
yield* TestClock.adjust(Duration.seconds(10));
2088+
const failed = yield* Fiber.join(refresh);
2089+
const expectedStatus = hasCachedRates ? "cached" : "unavailable";
2090+
assert.strictEqual(failed.status, expectedStatus);
2091+
assert.strictEqual(failed.fetchedAt, fetchedAt);
2092+
const requestsAfterFailure = requests;
2093+
assert.strictEqual(requestsAfterFailure, hasCachedRates ? 3 : 2);
2094+
2095+
responding = true;
2096+
const [immediate, summary] = yield* Effect.all(
2097+
[service.refreshRates, service.readSummary(WINDOW)],
2098+
{ concurrency: 2 },
2099+
);
2100+
assert.strictEqual(immediate.status, expectedStatus);
2101+
assert.strictEqual(summary.pricing.status, expectedStatus);
2102+
assert.strictEqual(requests, requestsAfterFailure);
2103+
yield* TestClock.adjust(Duration.seconds(59));
2104+
const coolingDown = yield* service.refreshRates;
2105+
assert.strictEqual(coolingDown.status, expectedStatus);
2106+
assert.strictEqual(coolingDown.fetchedAt, fetchedAt);
2107+
assert.strictEqual(requests, requestsAfterFailure);
2108+
2109+
yield* TestClock.adjust(Duration.seconds(1));
2110+
const recovered = yield* service.refreshRates;
2111+
assert.strictEqual(recovered.status, "fresh");
2112+
assert.strictEqual(recovered.knownModels, 1);
2113+
assert.notStrictEqual(recovered.fetchedAt, fetchedAt);
2114+
assert.strictEqual(requests, requestsAfterFailure + 1);
2115+
}).pipe(Effect.scoped, Effect.provide(TestClock.layer())),
2116+
);
2117+
20392118
it.live("does not orphan an in-flight scan when its first caller is interrupted", () =>
20402119
Effect.gen(function* () {
20412120
const { settings, home } = yield* setup;

‎apps/server/src/usage/UsageService.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,7 @@ export const make = Effect.gen(function* () {
212212
);
213213
let rates: RateTable = new Map();
214214
let ratesFetchedAtMs: number | null = null;
215+
let ratesLastFailedAtMs: number | null = null;
215216
let ratesStatus: UsagePricing["status"] = "unavailable";
216217
const ratesLock = yield* Semaphore.make(1);
217218

@@ -256,17 +257,21 @@ export const make = Effect.gen(function* () {
256257
}
257258
}
258259

260+
if (ratesLastFailedAtMs !== null && now - ratesLastFailedAtMs < RATES_REFRESH_FLOOR_MS) return;
261+
259262
let fetched = yield* fetchRates(LITELLM_RATES_URL);
260263
if (fetched === null) {
261264
fetched = yield* fetchRates(LITELLM_RATES_API_URL, "application/vnd.github.raw+json");
262265
}
263266
if (fetched === null) {
267+
ratesLastFailedAtMs = yield* Clock.currentTimeMillis;
264268
if (rates.size > 0) ratesStatus = "cached";
265269
return;
266270
}
267271

268272
rates = fetched.parsed;
269273
ratesFetchedAtMs = now;
274+
ratesLastFailedAtMs = null;
270275
ratesStatus = "fresh";
271276

272277
yield* encodeRatesCache({ fetchedAtMs: now, document: fetched.document }).pipe(

0 commit comments

Comments
 (0)