Skip to content

Commit bf5ee43

Browse files
committed
fix(server): discard usage successes invalidated in flight
1 parent 0e899b9 commit bf5ee43

2 files changed

Lines changed: 52 additions & 7 deletions

File tree

‎packages/provider-core/src/server/usageHttpClient.test.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ import { assert, it } from "@effect/vitest";
22
import * as DateTime from "effect/DateTime";
33
import * as Clock from "effect/Clock";
44
import * as Effect from "effect/Effect";
5+
import * as Deferred from "effect/Deferred";
6+
import * as Fiber from "effect/Fiber";
57
import * as TestClock from "effect/testing/TestClock";
68
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
79
import { makeUsageHttpClient, usageResponseCheckedAt } from "./usageHttpClient.ts";
@@ -83,3 +85,35 @@ it.effect.each(["seconds", "date", "missing"] as const)(
8385
assert.strictEqual(calls, 3);
8486
}),
8587
);
88+
89+
it.effect.each([200, 503, 429])(
90+
"invalidates in-flight successes without losing failed-read backoff (%s)",
91+
(status) =>
92+
Effect.gen(function* () {
93+
const started = yield* Deferred.make<void>();
94+
const finish = yield* Deferred.make<void>();
95+
let calls = 0;
96+
const client = yield* makeUsageHttpClient(
97+
HttpClient.make((input) =>
98+
Effect.gen(function* () {
99+
calls++;
100+
const reading = calls;
101+
if (reading === 1) {
102+
yield* Deferred.succeed(started, undefined);
103+
yield* Deferred.await(finish);
104+
}
105+
return HttpClientResponse.fromWeb(input, Response.json({ reading }, { status }));
106+
}),
107+
),
108+
);
109+
const pending = yield* client.execute(request("a")).pipe(Effect.forkChild);
110+
yield* Deferred.await(started);
111+
yield* client.invalidateSuccessfulReads("https://api.anthropic.com", "Bearer a");
112+
yield* Deferred.succeed(finish, undefined);
113+
yield* Fiber.join(pending);
114+
const refreshed = yield* client.execute(request("a"));
115+
const expected = status < 400 ? 2 : 1;
116+
assert.deepStrictEqual(yield* refreshed.json, { reading: expected });
117+
assert.strictEqual(calls, expected);
118+
}),
119+
);

‎packages/provider-core/src/server/usageHttpClient.ts‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,10 @@ export const makeUsageHttpClient = Effect.fn("makeUsageHttpClient")(function (
6262
readonly expiresAt: number;
6363
}
6464
>();
65-
const accounts = new Map<string, { readonly lock: Semaphore.Semaphore; retryAt: number }>();
65+
const accounts = new Map<
66+
string,
67+
{ readonly lock: Semaphore.Semaphore; retryAt: number; generation: number }
68+
>();
6669
const cachedClient = HttpClient.transform(client, (read, request) => {
6770
if (!isUsageUrl(request.url)) return read;
6871
return Effect.gen(function* () {
@@ -72,7 +75,7 @@ export const makeUsageHttpClient = Effect.fn("makeUsageHttpClient")(function (
7275
]);
7376
let account = accounts.get(accountKey);
7477
if (!account) {
75-
account = { lock: Semaphore.makeUnsafe(1), retryAt: 0 };
78+
account = { lock: Semaphore.makeUnsafe(1), retryAt: 0, generation: 0 };
7679
accounts.set(accountKey, account);
7780
}
7881
const state = account;
@@ -105,6 +108,7 @@ export const makeUsageHttpClient = Effect.fn("makeUsageHttpClient")(function (
105108
}),
106109
);
107110
}
111+
const generation = state.generation;
108112
const result = yield* read.pipe(
109113
Effect.flatMap((response) =>
110114
Effect.gen(function* () {
@@ -130,11 +134,16 @@ export const makeUsageHttpClient = Effect.fn("makeUsageHttpClient")(function (
130134
Effect.exit,
131135
);
132136
const finishedAt = yield* Clock.currentTimeMillis;
133-
cached.set(key, {
134-
accountKey,
135-
result,
136-
expiresAt: Math.max(finishedAt + TTL_MS, state.retryAt),
137-
});
137+
if (
138+
generation === state.generation ||
139+
Exit.isFailure(result) ||
140+
result.value.status >= 400
141+
)
142+
cached.set(key, {
143+
accountKey,
144+
result,
145+
expiresAt: Math.max(finishedAt + TTL_MS, state.retryAt),
146+
});
138147
const saved = yield* result;
139148
const response = HttpClientResponse.fromWeb(
140149
request,
@@ -150,6 +159,8 @@ export const makeUsageHttpClient = Effect.fn("makeUsageHttpClient")(function (
150159
invalidateSuccessfulReads: (origin: string, authorization: string) =>
151160
Effect.sync(() => {
152161
const accountKey = encodeKey([origin, authorization]);
162+
const account = accounts.get(accountKey);
163+
if (account) account.generation++;
153164
for (const [key, saved] of cached) {
154165
if (
155166
saved.accountKey === accountKey &&

0 commit comments

Comments
 (0)