Skip to content

Commit e1af0b2

Browse files
only21milclaudemaria-rcks
authored
perf(provider-core): provider instances share one latest-version lookup (#23763)
Co-authored-by: Claude Opus 5.5 <[email protected]> Co-authored-by: maria-rcks <[email protected]>
1 parent 8bc1891 commit e1af0b2

2 files changed

Lines changed: 179 additions & 16 deletions

File tree

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
import { describe, expect, it } from "@effect/vitest";
2+
import * as Deferred from "effect/Deferred";
3+
import * as Effect from "effect/Effect";
4+
import * as Exit from "effect/Exit";
5+
import * as Fiber from "effect/Fiber";
6+
7+
import * as ProviderLatestVersions from "./ProviderLatestVersions.ts";
8+
9+
describe("ProviderLatestVersions", () => {
10+
it.effect("concurrent misses for one key share a single lookup", () =>
11+
Effect.gen(function* () {
12+
const latestVersions = yield* ProviderLatestVersions.make();
13+
const release = yield* Deferred.make<void>();
14+
let lookups = 0;
15+
const lookup = Effect.suspend(() => {
16+
lookups += 1;
17+
return Deferred.await(release).pipe(Effect.as("2.0.0"));
18+
});
19+
20+
const callers = yield* Effect.forEach([1, 2, 3], () =>
21+
Effect.forkChild(latestVersions.cached("@example/cli", lookup), {
22+
startImmediately: true,
23+
}),
24+
);
25+
yield* Deferred.succeed(release, undefined);
26+
expect(yield* Effect.forEach(callers, Fiber.join)).toEqual(["2.0.0", "2.0.0", "2.0.0"]);
27+
expect(lookups).toBe(1);
28+
}),
29+
);
30+
31+
it.effect("a waiting caller looks up again when the shared lookup is interrupted", () =>
32+
Effect.gen(function* () {
33+
const latestVersions = yield* ProviderLatestVersions.make();
34+
let lookups = 0;
35+
const lookup = Effect.suspend(() => {
36+
lookups += 1;
37+
return lookups === 1 ? Effect.never : Effect.succeed("2.0.0");
38+
});
39+
40+
const owner = yield* Effect.forkChild(latestVersions.cached("@example/cli", lookup), {
41+
startImmediately: true,
42+
});
43+
const waiter = yield* Effect.forkChild(latestVersions.cached("@example/cli", lookup), {
44+
startImmediately: true,
45+
});
46+
expect(lookups).toBe(1);
47+
yield* Fiber.interrupt(owner);
48+
expect(yield* Fiber.join(waiter)).toBe("2.0.0");
49+
expect(yield* latestVersions.cached("@example/cli", lookup)).toBe("2.0.0");
50+
expect(lookups).toBe(2);
51+
}),
52+
);
53+
54+
it.effect("a caller interrupted while it claims the lookup frees the key", () =>
55+
Effect.gen(function* () {
56+
const latestVersions = yield* ProviderLatestVersions.make();
57+
let lookups = 0;
58+
const lookup = Effect.sync(() => {
59+
lookups += 1;
60+
return "2.0.0";
61+
});
62+
// `cached` reads `fresh` in the same step that claims the key, so the
63+
// interrupt arrives after the claim and before the lookup starts.
64+
const interruptOnClaim = {
65+
get fresh() {
66+
Fiber.getCurrent()?.interruptUnsafe();
67+
return false;
68+
},
69+
};
70+
71+
const owner = yield* Effect.forkChild(
72+
latestVersions.cached("@example/cli", lookup, interruptOnClaim),
73+
{ startImmediately: true },
74+
);
75+
expect(Exit.hasInterrupts(yield* Fiber.await(owner))).toBe(true);
76+
expect(lookups).toBe(0);
77+
const next = yield* Effect.forkChild(latestVersions.cached("@example/cli", lookup), {
78+
startImmediately: true,
79+
});
80+
expect(lookups).toBe(1);
81+
expect(yield* Fiber.join(next)).toBe("2.0.0");
82+
}),
83+
);
84+
it.effect("a waiting caller retries with its own lookup after the owner fails", () =>
85+
Effect.gen(function* () {
86+
const latestVersions = yield* ProviderLatestVersions.make();
87+
const release = yield* Deferred.make<void>();
88+
const owner = yield* Effect.forkChild(
89+
latestVersions.cached(
90+
"cli",
91+
Deferred.await(release).pipe(Effect.andThen(Effect.fail("offline"))),
92+
),
93+
{ startImmediately: true },
94+
);
95+
let retries = 0;
96+
const waiter = yield* Effect.forkChild(
97+
latestVersions.cached(
98+
"cli",
99+
Effect.sync(() => {
100+
retries += 1;
101+
return "2.0.0";
102+
}),
103+
),
104+
{ startImmediately: true },
105+
);
106+
expect(retries).toBe(0);
107+
yield* Deferred.succeed(release, undefined);
108+
expect(yield* Effect.flip(Fiber.join(owner))).toBe("offline");
109+
expect(yield* Fiber.join(waiter)).toBe("2.0.0");
110+
expect(retries).toBe(1);
111+
}),
112+
);
113+
114+
it.effect("interrupting a waiter leaves the shared lookup running", () =>
115+
Effect.gen(function* () {
116+
const latestVersions = yield* ProviderLatestVersions.make();
117+
const release = yield* Deferred.make<void>();
118+
let lookups = 0;
119+
const lookup = Effect.suspend(() => {
120+
lookups += 1;
121+
return Deferred.await(release).pipe(Effect.as("2.0.0"));
122+
});
123+
const owner = yield* Effect.forkChild(latestVersions.cached("cli", lookup), {
124+
startImmediately: true,
125+
});
126+
const waiter = yield* Effect.forkChild(latestVersions.cached("cli", lookup), {
127+
startImmediately: true,
128+
});
129+
yield* Fiber.interrupt(waiter);
130+
yield* Deferred.succeed(release, undefined);
131+
expect(yield* Fiber.join(owner)).toBe("2.0.0");
132+
expect(yield* latestVersions.cached("cli", lookup)).toBe("2.0.0");
133+
expect(lookups).toBe(1);
134+
}),
135+
);
136+
});

‎packages/provider-core/src/server/ProviderLatestVersions.ts‎

Lines changed: 43 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
11
import * as Context from "effect/Context";
22
import * as DateTime from "effect/DateTime";
3+
import * as Deferred from "effect/Deferred";
34
import * as Duration from "effect/Duration";
45
import * as Effect from "effect/Effect";
6+
import * as Exit from "effect/Exit";
57
import * as Layer from "effect/Layer";
8+
import * as Option from "effect/Option";
69
import * as Ref from "effect/Ref";
710

811
const FOUND_TTL = Duration.hours(1);
@@ -11,6 +14,7 @@ const MISSING_TTL = Duration.minutes(1);
1114
interface Entry {
1215
readonly expiresAt: number;
1316
readonly version: string | null;
17+
readonly inFlight?: Deferred.Deferred<Option.Option<string | null>>;
1418
}
1519

1620
export class ProviderLatestVersions extends Context.Service<
@@ -33,23 +37,46 @@ export const make = Effect.fn("ProviderLatestVersions.make")(function* (
3337
Array.from(seed, ([key, version]) => [key, { version, expiresAt: Number.MAX_SAFE_INTEGER }]),
3438
),
3539
);
40+
const cached: ProviderLatestVersions["Service"]["cached"] = Effect.fn(
41+
"ProviderLatestVersions.cached",
42+
)(function* (key, lookup, options) {
43+
const now = DateTime.toEpochMillis(yield* DateTime.now);
44+
const inFlight = yield* Deferred.make<Option.Option<string | null>>();
45+
// Replacing or invalidating this entry retires any lookup still in flight.
46+
const pending: Entry = { version: null, expiresAt: 0, inFlight };
47+
// Claim and cleanup registration must be atomic with respect to interruption.
48+
return yield* Effect.acquireUseRelease(
49+
Ref.modify(entries, (current) => {
50+
const entry = current.get(key);
51+
return !options?.fresh && entry && (entry.expiresAt > now || entry.inFlight)
52+
? [entry, current]
53+
: [undefined, new Map(current).set(key, pending)];
54+
}),
55+
(existing) =>
56+
Effect.gen(function* () {
57+
if (!existing) return yield* lookup;
58+
if (!existing.inFlight) return existing.version;
59+
const shared = yield* Deferred.await(existing.inFlight);
60+
return Option.isSome(shared) ? shared.value : yield* cached(key, lookup, options);
61+
}),
62+
(existing, exit) => {
63+
if (existing) return Effect.void;
64+
const version = Exit.isSuccess(exit) ? Option.some(exit.value) : Option.none();
65+
return Ref.update(entries, (current) => {
66+
if (current.get(key) !== pending) return current;
67+
const next = new Map(current);
68+
if (Option.isNone(version)) {
69+
next.delete(key);
70+
return next;
71+
}
72+
const ttl = version.value === null ? MISSING_TTL : FOUND_TTL;
73+
return next.set(key, { version: version.value, expiresAt: now + Duration.toMillis(ttl) });
74+
}).pipe(Effect.andThen(Deferred.succeed(inFlight, version)), Effect.asVoid);
75+
},
76+
);
77+
});
3678
return ProviderLatestVersions.of({
37-
cached: Effect.fn("ProviderLatestVersions.cached")(function* (key, lookup, options) {
38-
const now = DateTime.toEpochMillis(yield* DateTime.now);
39-
const existing = (yield* Ref.get(entries)).get(key);
40-
if (!options?.fresh && existing && existing.expiresAt > now) return existing.version;
41-
// Replacing or invalidating this entry retires any lookup still in flight.
42-
const pending = { version: null, expiresAt: 0 };
43-
yield* Ref.update(entries, (current) => new Map(current).set(key, pending));
44-
const version = yield* lookup;
45-
const ttl = version === null ? MISSING_TTL : FOUND_TTL;
46-
yield* Ref.update(entries, (current) =>
47-
current.get(key) === pending
48-
? new Map(current).set(key, { version, expiresAt: now + Duration.toMillis(ttl) })
49-
: current,
50-
);
51-
return version;
52-
}),
79+
cached,
5380
invalidate: Effect.fn("ProviderLatestVersions.invalidate")(function (key) {
5481
return Ref.update(entries, (current) => {
5582
const next = new Map(current);

0 commit comments

Comments
 (0)