Skip to content

Commit ab645b6

Browse files
committed
fix(usage): fall back to GitHub API for model prices
1 parent 16918b5 commit ab645b6

2 files changed

Lines changed: 183 additions & 18 deletions

File tree

‎apps/server/src/usage/UsageService.test.ts‎

Lines changed: 163 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ import * as Layer from "effect/Layer";
2929
import * as Scheduler from "effect/Scheduler";
3030
import * as Schema from "effect/Schema";
3131
import * as TestClock from "effect/testing/TestClock";
32-
import { HttpClient, HttpClientResponse } from "effect/http";
32+
import { HttpClient, HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/http";
3333

3434
import * as ServerConfig from "../config.ts";
3535
import * as ServerSettings from "../serverSettings.ts";
@@ -115,8 +115,12 @@ const layerService = (input: {
115115
readonly prefix: string;
116116
readonly home: string;
117117
readonly settings: Parameters<typeof ServerSettings.layerTest>[0];
118-
readonly onRatesFetch?: () => void;
118+
readonly onRatesFetch?: (request: HttpClientRequest.HttpClientRequest) => void;
119119
readonly ratesDocument?: unknown;
120+
readonly ratesStatus?: () => number;
121+
readonly ratesResponse?: (
122+
request: HttpClientRequest.HttpClientRequest,
123+
) => Effect.Effect<Response, HttpClientError.HttpClientError>;
120124
readonly environment?: NodeJS.ProcessEnv;
121125
readonly platform?: NodeJS.Platform;
122126
}) =>
@@ -137,9 +141,18 @@ const layerService = (input: {
137141
Layer.succeed(
138142
HttpClient.HttpClient,
139143
HttpClient.make((request) =>
140-
Effect.sync(() => {
141-
input.onRatesFetch?.();
142-
return HttpClientResponse.fromWeb(request, Response.json(input.ratesDocument ?? {}));
144+
Effect.suspend(() => {
145+
input.onRatesFetch?.(request);
146+
const response = input.ratesResponse
147+
? input.ratesResponse(request)
148+
: Effect.succeed(
149+
Response.json(input.ratesDocument ?? {}, {
150+
status: input.ratesStatus?.() ?? 200,
151+
}),
152+
);
153+
return response.pipe(
154+
Effect.map((response) => HttpClientResponse.fromWeb(request, response)),
155+
);
143156
}),
144157
),
145158
),
@@ -1834,10 +1847,10 @@ describe("UsageService", () => {
18341847
{ concurrency: 2 },
18351848
);
18361849
assert.deepStrictEqual(first, second);
1837-
assert.strictEqual(ratesFetches, 1);
1850+
assert.strictEqual(ratesFetches, 2);
18381851

18391852
yield* service.readSummary(WINDOW);
1840-
assert.strictEqual(ratesFetches, 2);
1853+
assert.strictEqual(ratesFetches, 4);
18411854
}).pipe(Effect.scoped),
18421855
);
18431856

@@ -1880,6 +1893,149 @@ describe("UsageService", () => {
18801893
}).pipe(Effect.scoped, Effect.provide(TestClock.layer())),
18811894
);
18821895

1896+
it.live("preserves cached estimates when both pricing downloads fail and recovers", () =>
1897+
Effect.gen(function* () {
1898+
const { transcript, settings, home } = yield* setup;
1899+
yield* Effect.promise(() => NodeFSP.writeFile(transcript, claudeLine(1, 5)));
1900+
let status = 200;
1901+
const requests: string[] = [];
1902+
const service = yield* UsageService.make.pipe(
1903+
Effect.provide(
1904+
layerService({
1905+
prefix: "usage-service-rates-failure-test",
1906+
home,
1907+
settings,
1908+
ratesDocument: {
1909+
"claude-fable-5": { input_cost_per_token: 1e-5, output_cost_per_token: 5e-5 },
1910+
},
1911+
ratesStatus: () => status,
1912+
onRatesFetch: (request) => requests.push(request.url),
1913+
}),
1914+
),
1915+
);
1916+
const first = yield* service.readSummary(WINDOW);
1917+
yield* TestClock.adjust(Duration.minutes(2));
1918+
status = 503;
1919+
1920+
const failed = yield* service.refreshRates;
1921+
assert.strictEqual(failed.status, "cached");
1922+
assert.strictEqual(failed.fetchedAt, first.pricing.fetchedAt);
1923+
assert.strictEqual(requests.length, 3);
1924+
assert.strictEqual(
1925+
requests[2],
1926+
"https://api.github.com/repos/BerriAI/litellm/contents/model_prices_and_context_window.json?ref=main",
1927+
);
1928+
const cached = yield* service.readSummary(WINDOW);
1929+
assert.deepStrictEqual(cached.buckets, first.buckets);
1930+
1931+
status = 200;
1932+
const recovered = yield* service.refreshRates;
1933+
assert.strictEqual(recovered.status, "fresh");
1934+
assert.notStrictEqual(recovered.fetchedAt, first.pricing.fetchedAt);
1935+
}).pipe(Effect.scoped, Effect.provide(TestClock.layer())),
1936+
);
1937+
1938+
it.live.each(["network", "http", "invalidJson", "empty"] as const)(
1939+
"recovers pricing through the GitHub API after a %s failure",
1940+
(failure) =>
1941+
Effect.gen(function* () {
1942+
const { transcript, settings, home } = yield* setup;
1943+
yield* Effect.promise(() => NodeFSP.writeFile(transcript, claudeLine(1, 5)));
1944+
const requests: HttpClientRequest.HttpClientRequest[] = [];
1945+
const ratesDocument = {
1946+
"claude-fable-5": { input_cost_per_token: 1e-5, output_cost_per_token: 5e-5 },
1947+
};
1948+
const serviceLayer = layerService({
1949+
prefix: `usage-service-rates-fallback-${failure}-test`,
1950+
home,
1951+
settings,
1952+
onRatesFetch: (request) => requests.push(request),
1953+
ratesResponse: (request) => {
1954+
if (request.url.startsWith("https://api.github.com/")) {
1955+
return Effect.succeed(Response.json(ratesDocument));
1956+
}
1957+
if (failure === "network") {
1958+
return Effect.fail(
1959+
new HttpClientError.HttpClientError({
1960+
reason: new HttpClientError.TransportError({
1961+
request,
1962+
cause: new Error("getaddrinfo ENOTFOUND raw.githubusercontent.com"),
1963+
}),
1964+
}),
1965+
);
1966+
}
1967+
return Effect.succeed(
1968+
failure === "invalidJson"
1969+
? new Response("not JSON")
1970+
: Response.json({}, { status: failure === "http" ? 503 : 200 }),
1971+
);
1972+
},
1973+
});
1974+
const service = yield* UsageService.make.pipe(Effect.provide(serviceLayer));
1975+
1976+
const summary = yield* service.readSummary(WINDOW);
1977+
assert.strictEqual(summary.pricing.status, "fresh");
1978+
assert.strictEqual(summary.pricing.knownModels, 1);
1979+
assert.closeTo(summary.buckets[0]?.costUsd ?? -1, 0.00035, 1e-12);
1980+
assert.deepStrictEqual(
1981+
requests.map((request) => request.url),
1982+
[
1983+
"https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json",
1984+
"https://api.github.com/repos/BerriAI/litellm/contents/model_prices_and_context_window.json?ref=main",
1985+
],
1986+
);
1987+
assert.strictEqual(requests[1]?.headers.accept, "application/vnd.github.raw+json");
1988+
1989+
const restarted = yield* UsageService.make.pipe(Effect.provide(serviceLayer));
1990+
const cached = yield* restarted.readSummary(WINDOW);
1991+
assert.strictEqual(cached.pricing.status, "cached");
1992+
assert.deepStrictEqual(cached.buckets, summary.buckets);
1993+
assert.strictEqual(requests.length, 2);
1994+
}).pipe(Effect.scoped),
1995+
);
1996+
1997+
it.live("tries the GitHub API after the primary pricing request times out", () =>
1998+
Effect.gen(function* () {
1999+
const { settings, home } = yield* setup;
2000+
const primaryStarted = yield* Deferred.make<void>();
2001+
const service = yield* UsageService.make.pipe(
2002+
Effect.provide(
2003+
layerService({
2004+
prefix: "usage-service-rates-timeout-test",
2005+
home,
2006+
settings,
2007+
ratesResponse: (request) =>
2008+
request.url.startsWith("https://api.github.com/")
2009+
? Effect.succeed(
2010+
Response.json({
2011+
"claude-fable-5": { input_cost_per_token: 1e-5, output_cost_per_token: 5e-5 },
2012+
}),
2013+
)
2014+
: Deferred.succeed(primaryStarted, undefined).pipe(Effect.andThen(Effect.never)),
2015+
}),
2016+
),
2017+
);
2018+
const refresh = yield* service.refreshRates.pipe(Effect.forkChild);
2019+
yield* Deferred.await(primaryStarted);
2020+
yield* TestClock.adjust(Duration.seconds(10));
2021+
const pricing = yield* Fiber.join(refresh);
2022+
assert.strictEqual(pricing.status, "fresh");
2023+
assert.strictEqual(pricing.knownModels, 1);
2024+
}).pipe(Effect.scoped, Effect.provide(TestClock.layer())),
2025+
);
2026+
2027+
it.live("keeps pricing unavailable when neither download has usable rates", () =>
2028+
Effect.gen(function* () {
2029+
const { settings, home } = yield* setup;
2030+
const service = yield* UsageService.make.pipe(
2031+
Effect.provide(layerService({ prefix: "usage-service-rates-empty-test", home, settings })),
2032+
);
2033+
const pricing = yield* service.refreshRates;
2034+
assert.strictEqual(pricing.status, "unavailable");
2035+
assert.isNull(pricing.fetchedAt);
2036+
}).pipe(Effect.scoped),
2037+
);
2038+
18832039
it.live("does not orphan an in-flight scan when its first caller is interrupted", () =>
18842040
Effect.gen(function* () {
18852041
const { settings, home } = yield* setup;

‎apps/server/src/usage/UsageService.ts‎

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@ import {
6060

6161
const LITELLM_RATES_URL =
6262
"https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json";
63+
const LITELLM_RATES_API_URL =
64+
"https://api.github.com/repos/BerriAI/litellm/contents/model_prices_and_context_window.json?ref=main";
6365

6466
const RATES_TTL_MS = 24 * 60 * 60 * 1000;
6567

@@ -221,6 +223,18 @@ export const make = Effect.gen(function* () {
221223
knownModels: rates.size,
222224
});
223225

226+
const fetchRates = (url: string, accept = "application/json") =>
227+
httpClient.get(url, { headers: { accept } }).pipe(
228+
Effect.flatMap(HttpClientResponse.filterStatusOk),
229+
Effect.flatMap((response) => response.json),
230+
Effect.map((document) => {
231+
const parsed = parseRateTable(document);
232+
return parsed.size > 0 ? { document, parsed } : null;
233+
}),
234+
Effect.timeout(10_000),
235+
Effect.catchCause(() => Effect.succeed(null)),
236+
);
237+
224238
const loadRates = Effect.fn("UsageService.loadRates")(function* (force: boolean) {
225239
const now = yield* Clock.currentTimeMillis;
226240
const maxAgeMs = force ? RATES_REFRESH_FLOOR_MS : RATES_TTL_MS;
@@ -242,25 +256,20 @@ export const make = Effect.gen(function* () {
242256
}
243257
}
244258

245-
const fetched = yield* httpClient.get(LITELLM_RATES_URL).pipe(
246-
Effect.flatMap(HttpClientResponse.filterStatusOk),
247-
Effect.flatMap((response) => response.json),
248-
Effect.timeout(10_000),
249-
Effect.catchCause(() => Effect.succeed(null)),
250-
);
259+
let fetched = yield* fetchRates(LITELLM_RATES_URL);
260+
if (fetched === null) {
261+
fetched = yield* fetchRates(LITELLM_RATES_API_URL, "application/vnd.github.raw+json");
262+
}
251263
if (fetched === null) {
252264
if (rates.size > 0) ratesStatus = "cached";
253265
return;
254266
}
255267

256-
const parsed = parseRateTable(fetched);
257-
if (parsed.size === 0) return;
258-
259-
rates = parsed;
268+
rates = fetched.parsed;
260269
ratesFetchedAtMs = now;
261270
ratesStatus = "fresh";
262271

263-
yield* encodeRatesCache({ fetchedAtMs: now, document: fetched }).pipe(
272+
yield* encodeRatesCache({ fetchedAtMs: now, document: fetched.document }).pipe(
264273
Effect.flatMap((contents) => writeCacheFile(ratesCachePath, contents)),
265274
Effect.ignoreCause,
266275
);

0 commit comments

Comments
 (0)