Skip to content

Commit 658688a

Browse files
committed
perf(cursor): improve account usage scanning
- Limit concurrent account refreshes and prune expired cached records - Avoid hashing disjoint pages while preserving overlap deduplication
1 parent bbdc7dc commit 658688a

6 files changed

Lines changed: 133 additions & 53 deletions

File tree

‎apps/server/src/usage/UsageService.test.ts‎

Lines changed: 30 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1580,24 +1580,37 @@ describe("UsageService", () => {
15801580
Effect.gen(function* () {
15811581
const { settings, home } = yield* setup;
15821582
const cursor = makeFakeCursor();
1583+
let inFlight = 0;
1584+
let mostInFlight = 0;
15831585
const started = yield* Deferred.make<void>();
15841586
const gate = yield* Deferred.make<void>();
15851587
yield* Effect.gen(function* () {
15861588
const context = yield* Layer.build(
15871589
layerCursorUsageAccounts(
15881590
Layer.succeed(CursorAccountReader.CursorAccountReader, {
15891591
read: (credential, sinceMs, untilMs) =>
1590-
(credential as { apiKey?: string }).apiKey === "failing-key"
1591-
? Effect.succeed({
1592+
Effect.gen(function* () {
1593+
inFlight++;
1594+
mostInFlight = Math.max(mostInFlight, inFlight);
1595+
if (credential.apiKey === "failing-key") {
1596+
yield* Deferred.await(started);
1597+
return {
15921598
accountKey: null,
15931599
records: [],
15941600
missing: false,
15951601
error: "Cursor account usage could not be read.",
1596-
})
1597-
: Deferred.succeed(started, undefined).pipe(
1598-
Effect.andThen(Deferred.await(gate)),
1599-
Effect.andThen(cursor.read(credential, sinceMs, untilMs)),
1600-
),
1602+
};
1603+
}
1604+
yield* Deferred.succeed(started, undefined);
1605+
yield* Deferred.await(gate);
1606+
return yield* cursor.read(credential, sinceMs, untilMs);
1607+
}).pipe(
1608+
Effect.ensuring(
1609+
Effect.sync(() => {
1610+
inFlight--;
1611+
}),
1612+
),
1613+
),
16011614
}),
16021615
),
16031616
);
@@ -1613,6 +1626,14 @@ describe("UsageService", () => {
16131626
instanceId: ProviderInstanceId.make("cursor-work"),
16141627
environment: { CURSOR_API_KEY: "new-key" },
16151628
},
1629+
{
1630+
instanceId: ProviderInstanceId.make("cursor-third"),
1631+
environment: { CURSOR_API_KEY: "third-key" },
1632+
},
1633+
{
1634+
instanceId: ProviderInstanceId.make("cursor-fourth"),
1635+
environment: { CURSOR_API_KEY: "fourth-key" },
1636+
},
16161637
],
16171638
windowStartMs: 0,
16181639
retentionCutoffMs: 0,
@@ -1624,6 +1645,8 @@ describe("UsageService", () => {
16241645
assert.isTrue(sources.some((source) => source.refreshing === true));
16251646
yield* Deferred.succeed(gate, undefined);
16261647
yield* Fiber.join(first);
1648+
assert.strictEqual(mostInFlight, 2);
1649+
assert.strictEqual(inFlight, 0);
16271650
}).pipe(
16281651
Effect.provide(
16291652
layerService({

‎packages/provider-cursor/src/server/CursorUsageAccounts.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ const make = Effect.gen(function* () {
104104
);
105105
yield* Effect.addFinalizer(() => awaitPersisted);
106106

107+
const refreshLock = yield* Semaphore.make(2);
107108
const refreshAccount = Effect.fn("CursorUsageAccounts.refreshAccount")(function* (
108109
credential: CursorCredentialSource,
109110
credentialKey: string,
@@ -149,7 +150,7 @@ const make = Effect.gen(function* () {
149150
failures.delete(credentialKey);
150151
if (merged.changed) cacheDirty = true;
151152
yield* schedulePersist;
152-
});
153+
}, refreshLock.withPermit);
153154

154155
const startRefresh = (
155156
credential: CursorCredentialSource,
@@ -256,6 +257,7 @@ const make = Effect.gen(function* () {
256257
retentionCutoffMs,
257258
awaitRefresh,
258259
),
260+
{ concurrency: 2 },
259261
);
260262
// Keys for one account share its source, and a repeated source would count sessions twice.
261263
// A failed key can hold older records, so a current read wins, then a refresh the client

‎packages/provider-cursor/src/server/accountCache.test.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,16 @@ describe("cursorAccountCache", () => {
6565
assert.isFalse(
6666
mergeCursorFetch(cache, "a", range, [record(10 * DAY_MS)], 11 * DAY_MS, 0).changed,
6767
);
68+
const pruned = mergeCursorFetch(
69+
cache,
70+
"a",
71+
range,
72+
[record(10 * DAY_MS)],
73+
11 * DAY_MS,
74+
2 * DAY_MS,
75+
);
76+
assert.isTrue(pruned.changed);
77+
assert.deepStrictEqual(pruned.cache.records, [record(10 * DAY_MS)]);
6878
assert.isTrue(
6979
mergeCursorFetch(
7080
cache,

‎packages/provider-cursor/src/server/accountCache.ts‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,22 +53,30 @@ export function mergeCursorFetch(
5353
record.timestampMs >= range.sinceMs && record.timestampMs <= range.untilMs;
5454
const previous = cache?.records ?? [];
5555
const added = fetched.filter(inRange);
56-
const replacedKeys = new Set(previous.filter(inRange).map((record) => record.dedupeKey));
56+
const replacedKeys = new Set<string | null>();
57+
let pruned = false;
58+
for (const record of previous) {
59+
if (inRange(record)) replacedKeys.add(record.dedupeKey);
60+
else if (record.timestampMs < retentionStartMs) pruned = true;
61+
}
5762
const changed =
5863
cache === undefined ||
64+
pruned ||
5965
replacedKeys.size !== added.length ||
6066
added.some((record) => !replacedKeys.has(record.dedupeKey));
67+
const records = !changed
68+
? previous
69+
: previous
70+
.filter((record) => !inRange(record) && record.timestampMs >= retentionStartMs)
71+
.concat(added);
6172
return {
6273
changed,
6374
cache: {
6475
accountKey,
6576
sinceMs: retentionStartMs,
6677
untilMs: range.untilMs,
6778
fetchedAtMs: nowMs,
68-
records: [
69-
...previous.filter((record) => !inRange(record) && record.timestampMs >= retentionStartMs),
70-
...added,
71-
],
79+
records,
7280
} satisfies CursorAccountCache,
7381
};
7482
}

‎packages/provider-cursor/src/server/accountUsage.test.ts‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,10 @@ describe("Cursor account history", () => {
115115
reasoningTokens: 0,
116116
});
117117
assert.strictEqual(result.records[0]?.reportedCostUsd, 0.25);
118+
assert.strictEqual(
119+
result.records[0]?.dedupeKey,
120+
"cursor-account:2188c26da759c4fae3aa0252fc6d725eff0922ef0b022551e6f0987d02bfab33:3149aaf8419ad6b66ef7330853d8fe85e6f2e9876e2ad5e85f88bee8c8d6d133:0",
121+
);
118122
assert.isFalse(result.accountKey?.includes("demo") ?? true);
119123
}).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
120124
);
@@ -209,7 +213,8 @@ describe("Cursor account history", () => {
209213
model: "gpt-5",
210214
tokenUsage: { inputTokens: 10, outputTokens: 5, totalCents: 1 },
211215
});
212-
for (const total of [2000, 2001]) {
216+
for (const total of [2000, 2001, undefined]) {
217+
const expectedCount = total ?? 2001;
213218
let requests = 0;
214219
const result = yield* readAccountUsage(credential, 0, 1781000000000, async () => {
215220
requests++;
@@ -219,19 +224,33 @@ describe("Cursor account history", () => {
219224
requests === 1
220225
? Array.from({ length: 1000 }, (_, index) => event(index))
221226
: requests === 2
222-
? Array.from({ length: 1000 }, (_, index) => event(999 + index))
227+
? Array.from({ length: 1000 }, (_, index) => {
228+
const row = event(999 + index);
229+
return {
230+
tokenUsage: {
231+
totalCents: row.tokenUsage.totalCents,
232+
outputTokens: row.tokenUsage.outputTokens,
233+
inputTokens: row.tokenUsage.inputTokens,
234+
},
235+
model: row.model,
236+
timestamp: row.timestamp,
237+
};
238+
})
223239
: [event(1999)],
224240
});
225241
});
226242
assert.isNull(result.error);
227-
assert.strictEqual(result.records.length, total);
243+
assert.strictEqual(result.records.length, expectedCount);
228244
assert.strictEqual(requests, 3);
229245
assert.strictEqual(result.records.at(-1)?.timestampMs, 1780000001999);
230246
assert.strictEqual(
231247
result.records.filter((record) => record.timestampMs === 1780000000999).length,
232248
total === 2000 ? 1 : 2,
233249
);
234-
assert.strictEqual(new Set(result.records.map((record) => record.dedupeKey)).size, total);
250+
assert.strictEqual(
251+
new Set(result.records.map((record) => record.dedupeKey)).size,
252+
expectedCount,
253+
);
235254
}
236255
let requests = 0;
237256
const inconsistent = yield* readAccountUsage(credential, 0, 1781000000000, async () => {

‎packages/provider-cursor/src/server/accountUsage.ts‎

Lines changed: 54 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ import * as Digest from "effect/crypto/Digest";
33
import * as Duration from "effect/Duration";
44
import * as Effect from "effect/Effect";
55
import * as Base64Url from "effect/encoding/Base64Url";
6-
import * as Hex from "effect/encoding/Hex";
76
import * as Fiber from "effect/Fiber";
87
import * as FetchHttpClient from "effect/http/FetchHttpClient";
98
import * as HttpClient from "effect/http/HttpClient";
@@ -55,12 +54,6 @@ const billedEventIdentity = (...fields: readonly unknown[]) => JSON.stringify(fi
5554

5655
const textEncoder = new TextEncoder();
5756

58-
// Untraced: each usage scan hashes every billed event.
59-
const accountHash = Effect.fnUntraced(function* (value: string) {
60-
const digest = yield* Digest.Digest;
61-
return Hex.encode(yield* digest.digest(Digest.sha256, textEncoder.encode(value)));
62-
});
63-
6457
function canonicalJson(value: unknown): string {
6558
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
6659
if (value !== null && typeof value === "object") {
@@ -93,6 +86,9 @@ export const readCursorAccountUsage = Effect.fn("readCursorAccountUsage")(functi
9386
endDate: number,
9487
): Effect.fn.Return<CursorAccountUsageReadResult, never, Digest.Digest | HttpClient.HttpClient> {
9588
const httpClient = yield* HttpClient.HttpClient;
89+
const digest = yield* Digest.Digest;
90+
// Per-event hashes share the scan span and use the native hex encoder.
91+
const accountHash = (value: string) => digest.digestHex(Digest.sha256, textEncoder.encode(value));
9692
const credential = yield* Effect.result(exchangeCursorApiKey(credentialSource.apiKey));
9793
if (Result.isFailure(credential)) {
9894
return {
@@ -124,8 +120,9 @@ export const readCursorAccountUsage = Effect.fn("readCursorAccountUsage")(functi
124120
);
125121
const records: UsageRecord[] = [];
126122
const occurrences = new Map<string, number>();
127-
const pages: unknown[][] = [];
128-
let completed = false;
123+
const normalized: Array<(Omit<UsageRecord, "dedupeKey"> & { dedupeKey: string }) | null> = [];
124+
const pages: Array<{ start: number; end: number; overlap: number }> = [];
125+
let previousEvents: unknown[] = [];
129126
const pageSize = 1000;
130127
let total: number | undefined;
131128
const readPage = Effect.fn("CursorAccountUsage.readPage")(
@@ -210,30 +207,23 @@ export const readCursorAccountUsage = Effect.fn("readCursorAccountUsage")(functi
210207
const next = ahead.shift();
211208
const events = yield* next === undefined ? readPage(page) : Fiber.join(next);
212209
if (!Array.isArray(events)) return events;
213-
pages.push(events);
214-
if (events.length < pageSize) {
215-
completed = true;
216-
break;
217-
}
218-
}
219-
if (!completed) return yield* invalid("Account usage page limit exceeded");
220-
const rawCount = pages.reduce((sum, page) => sum + page.length, 0);
221-
if (total !== undefined && rawCount < total)
222-
return yield* invalid("Incomplete account usage pages");
223-
let removalsRemaining = total === undefined ? 0 : rawCount - total;
224-
let previousKeys: string[] = [];
225-
for (const events of pages) {
226-
const eventKeys =
227-
removalsRemaining > 0
228-
? yield* Effect.forEach(events, (event) => accountHash(canonicalJson(event)))
229-
: [];
230-
const removalCount = Math.min(removalsRemaining, boundaryOverlap(previousKeys, eventKeys));
231-
removalsRemaining -= removalCount;
232-
previousKeys = eventKeys;
233-
for (const raw of events.slice(removalCount)) {
210+
const start = normalized.length;
211+
const firstTimestamp = object(events[0]).timestamp;
212+
// Equal raw events must have equal timestamps; avoid canonicalizing disjoint pages.
213+
const mayOverlap =
214+
(typeof firstTimestamp !== "string" && typeof firstTimestamp !== "number") ||
215+
previousEvents.some((event) => object(event).timestamp === firstTimestamp);
216+
const overlap = mayOverlap
217+
? boundaryOverlap(previousEvents.map(canonicalJson), events.map(canonicalJson))
218+
: 0;
219+
previousEvents = events;
220+
for (const raw of events) {
234221
const event = object(raw);
235222
const usage = object(event.tokenUsage);
236-
if (event.tokenUsage === undefined || event.tokenUsage === null) continue;
223+
if (event.tokenUsage === undefined || event.tokenUsage === null) {
224+
normalized.push(null);
225+
continue;
226+
}
237227
for (const key of [
238228
"inputTokens",
239229
"outputTokens",
@@ -260,7 +250,10 @@ export const readCursorAccountUsage = Effect.fn("readCursorAccountUsage")(functi
260250
!event.model
261251
)
262252
return yield* invalid("Invalid account usage event");
263-
if (timestampMs < sinceMs || timestampMs > endDate) continue;
253+
if (timestampMs < sinceMs || timestampMs > endDate) {
254+
normalized.push(null);
255+
continue;
256+
}
264257
const totals = {
265258
uncachedInputTokens: tokens(usage.inputTokens),
266259
cachedInputTokens: tokens(usage.cacheReadTokens),
@@ -275,9 +268,7 @@ export const readCursorAccountUsage = Effect.fn("readCursorAccountUsage")(functi
275268
const key = yield* accountHash(
276269
billedEventIdentity(timestampMs, event.model, sessionId, totals, reportedCostUsd),
277270
);
278-
const occurrence = occurrences.get(key) ?? 0;
279-
occurrences.set(key, occurrence + 1);
280-
records.push({
271+
normalized.push({
281272
provider: "cursor",
282273
timestampMs,
283274
model: event.model,
@@ -286,12 +277,39 @@ export const readCursorAccountUsage = Effect.fn("readCursorAccountUsage")(functi
286277
totals,
287278
reportedCostUsd,
288279
speed: "standard",
289-
dedupeKey: `cursor-account:${accountKey}:${key}:${occurrence}`,
280+
dedupeKey: key,
290281
});
291282
}
283+
pages.push({ start, end: normalized.length, overlap });
284+
if (events.length < pageSize) break;
285+
yield* Effect.yieldNow;
286+
}
287+
const rawCount = normalized.length;
288+
if (total !== undefined && rawCount < total)
289+
return yield* invalid("Incomplete account usage pages");
290+
let removalsRemaining = total === undefined ? 0 : rawCount - total;
291+
// The final count proves which boundary copies to remove, including non-token events.
292+
for (const page of pages) {
293+
const removalCount = Math.min(removalsRemaining, page.overlap);
294+
removalsRemaining -= removalCount;
295+
for (let index = page.start; index < page.end; index++) {
296+
const record = normalized[index];
297+
normalized[index] = null;
298+
if (index < page.start + removalCount || record == null) continue;
299+
const key = record.dedupeKey;
300+
const occurrence = occurrences.get(key) ?? 0;
301+
occurrences.set(key, occurrence + 1);
302+
record.dedupeKey = `cursor-account:${accountKey}:${key}:${occurrence}`;
303+
records.push(record);
304+
}
292305
yield* Effect.yieldNow;
293306
}
294307
if (removalsRemaining !== 0) return yield* invalid("Inconsistent account usage boundaries");
308+
yield* Effect.annotateCurrentSpan({
309+
"cursor.usage.pages": pages.length,
310+
"cursor.usage.events": rawCount,
311+
"cursor.usage.records": records.length,
312+
});
295313
return {
296314
accountKey,
297315
records,

0 commit comments

Comments
 (0)