Skip to content

Commit f6674db

Browse files
refactor(usage): answer provisional reads only from the saved scan
Drop the in-memory map of the last 32 exact summaries, so a provisional read always goes through directory coverage. Decide coverage before the scan starts: a covered read prices with the saved rates and never takes the rates lock, anything else is a normal exact scan. A missing directory now counts as covered, which removes the existence probe and the three-way coverage state. Sources are marked refreshing while they are built instead of in a second pass over the summary. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent a32cedb commit f6674db

1 file changed

Lines changed: 80 additions & 153 deletions

File tree

‎apps/server/src/usage/UsageService.ts‎

Lines changed: 80 additions & 153 deletions
Original file line numberDiff line numberDiff line change
@@ -168,11 +168,15 @@ export const make = Effect.gen(function* () {
168168
const correctedBatch = (records: readonly UsageRecord[]) =>
169169
records.some((record) => resolvedRecords.has(record)) ? records.map(correctedRecord) : records;
170170
const sourceCache = new Map<string, typeof CachedSource.Type>();
171-
// Earliest window start a full scan read each transcript directory from. A window inside it
172-
// has all its files in fileCache except what changed since, so it can be answered from memory.
171+
// Earliest window start a complete scan read each transcript directory from.
173172
const coveredSince = new Map<string, number>();
174173
const coverageKey = (source: TranscriptSource) => `${source.provider}\0${source.dir}`;
175174
let cacheDirty = false;
175+
const cover = (source: TranscriptSource, windowStartMs: number) => {
176+
if ((coveredSince.get(coverageKey(source)) ?? Infinity) <= windowStartMs) return;
177+
coveredSince.set(coverageKey(source), windowStartMs);
178+
cacheDirty = true;
179+
};
176180
const isWithinDirectory = (filePath: string, dir: string) => {
177181
const relative = path.relative(dir, filePath);
178182
return relative !== ".." && !relative.startsWith(".." + path.sep) && !path.isAbsolute(relative);
@@ -242,32 +246,26 @@ export const make = Effect.gen(function* () {
242246
Effect.catchCause(() => Effect.succeed(null)),
243247
);
244248

245-
const loadRates = Effect.fn("UsageService.loadRates")(function* (
246-
force: boolean,
247-
savedOnly = false,
248-
) {
249+
// Outside ratesLock, which a slow fetch holds: a provisional answer prices with these.
250+
const loadSavedRates = yield* Effect.cached(
251+
fileSystem.readFileString(ratesCachePath).pipe(
252+
Effect.flatMap((raw) => decodeRatesCache(raw)),
253+
Effect.map((saved) => {
254+
const parsed = parseRateTable(saved.document);
255+
if (parsed.size === 0) return;
256+
rates = parsed;
257+
ratesFetchedAtMs = saved.fetchedAtMs;
258+
ratesStatus = "cached";
259+
}),
260+
Effect.ignoreCause,
261+
),
262+
);
263+
264+
const loadRates = Effect.fn("UsageService.loadRates")(function* (force: boolean) {
249265
const now = yield* Clock.currentTimeMillis;
250266
const maxAgeMs = force ? RATES_REFRESH_FLOOR_MS : RATES_TTL_MS;
267+
yield* loadSavedRates;
251268
if (ratesFetchedAtMs !== null && now - ratesFetchedAtMs < maxAgeMs) return;
252-
253-
if (ratesFetchedAtMs === null) {
254-
const fromDisk = yield* fileSystem.readFileString(ratesCachePath).pipe(
255-
Effect.flatMap((raw) => decodeRatesCache(raw)),
256-
Effect.catchCause(() => Effect.succeed(null)),
257-
);
258-
if (fromDisk !== null) {
259-
const parsed = parseRateTable(fromDisk.document);
260-
if (parsed.size > 0) {
261-
rates = parsed;
262-
ratesFetchedAtMs = fromDisk.fetchedAtMs;
263-
ratesStatus = "cached";
264-
if (now - fromDisk.fetchedAtMs < maxAgeMs) return;
265-
}
266-
}
267-
}
268-
269-
// Saved rates of any age price a provisional answer; only an exact read waits on the network.
270-
if (savedOnly && rates.size > 0) return;
271269
if (ratesLastFailedAtMs !== null && now - ratesLastFailedAtMs < RATES_REFRESH_FLOOR_MS) return;
272270

273271
let fetched = yield* fetchRates(LITELLM_RATES_URL);
@@ -291,8 +289,7 @@ export const make = Effect.gen(function* () {
291289
);
292290
});
293291

294-
const ensureRates = (force: boolean, savedOnly = false) =>
295-
ratesLock.withPermit(loadRates(force, savedOnly));
292+
const ensureRates = (force: boolean) => ratesLock.withPermit(loadRates(force));
296293

297294
const refreshRates = ensureRates(true).pipe(
298295
Effect.map(pricing),
@@ -628,8 +625,11 @@ export const make = Effect.gen(function* () {
628625
const exists = yield* fileSystem
629626
.exists(dir)
630627
.pipe(Effect.catchCause(() => Effect.succeed(false)));
631-
if (!exists) return { provider, dir, volumeId, files: null } satisfies ScannedDir;
632-
// No walk: scanSummary aggregates this directory's saved files, as it does for deleted ones.
628+
if (!exists) {
629+
cover(source, windowStartMs);
630+
return { provider, dir, volumeId, files: null } satisfies ScannedDir;
631+
}
632+
// scanSummary aggregates the saved files, as it does for a deleted directory.
633633
if (fromMemory) return { provider, dir, volumeId, files: [] } satisfies ScannedDir;
634634
const { files, failedPaths } = yield* Effect.promise(() =>
635635
listTranscriptFiles(dir, windowStartMs, fileName === undefined ? undefined : { fileName }),
@@ -663,12 +663,7 @@ export const make = Effect.gen(function* () {
663663
return { path, records, malformedRecords };
664664
});
665665
const unread = failedPaths + read.filter((file) => file.failed).length;
666-
const covered = coveredSince.get(coverageKey(source));
667-
// A read that missed paths covers nothing: memory would answer for files it never held.
668-
if (unread === 0 && (covered === undefined || covered > windowStartMs)) {
669-
coveredSince.set(coverageKey(source), windowStartMs);
670-
cacheDirty = true;
671-
}
666+
if (unread === 0) cover(source, windowStartMs);
672667
yield* Effect.annotateCurrentSpan({
673668
"usage.files": files.length,
674669
"usage.files_parsed": read.filter((file) => file.update !== undefined).length,
@@ -688,34 +683,13 @@ export const make = Effect.gen(function* () {
688683
});
689684

690685
const collectDirs = Effect.fn("UsageService.collectDirs")(function* (
686+
{ dirs, pending }: Effect.Success<ReturnType<typeof resolveTranscriptDirs>>,
691687
windowStartMs: number,
692688
settings: ServerSettingsValue,
693689
retentionCutoffMs: number,
694690
awaitRefresh: boolean,
695-
provisional: boolean,
691+
fromMemory: boolean,
696692
) {
697-
const { dirs, pending } = yield* resolveTranscriptDirs(
698-
settings,
699-
retentionCutoffMs,
700-
awaitRefresh,
701-
).pipe(Effect.provideContext(readerContext));
702-
// A window no full scan has read would show partial totals, so every directory must cover it.
703-
const coverage = provisional
704-
? yield* Effect.forEach(dirs, (source) =>
705-
fileSystem.exists(source.dir).pipe(
706-
Effect.catchCause(() => Effect.succeed(false)),
707-
Effect.map((exists) =>
708-
!exists
709-
? "missing"
710-
: (coveredSince.get(coverageKey(source)) ?? Infinity) <= windowStartMs
711-
? "covered"
712-
: "uncovered",
713-
),
714-
),
715-
)
716-
: [];
717-
const fromMemory = coverage.includes("covered") && !coverage.includes("uncovered");
718-
719693
const scans = Effect.forEach(
720694
scanReaders,
721695
({ driver, reader }) =>
@@ -749,36 +723,9 @@ export const make = Effect.gen(function* () {
749723
{ concurrency: "unbounded" },
750724
);
751725
const scanned: readonly ScannedDir[] = [...transcripts, ...pending, ...scanDirs.flat()];
752-
return { scanned, fromMemory, unread: transcripts.some((dir) => "status" in dir) };
726+
return scanned;
753727
});
754728

755-
/**
756-
* A saved answer misses whatever changed since its scan, so it is never final: its transcript
757-
* sources with usage are marked refreshing, which tells the client to ask again with
758-
* awaitRefresh. Null when none has usage: there is nothing to show early, and marking an
759-
* empty source would only flash a loading row.
760-
*/
761-
const asSaved = (summary: UsageSummary): UsageSummary | null => {
762-
const marked = new Set(
763-
summary.sources.filter(
764-
({ fingerprint }) =>
765-
transcriptFormats.has(fingerprint.provider) &&
766-
summary.buckets.some(
767-
(bucket) =>
768-
bucket.provider === fingerprint.provider &&
769-
bucket.sourcePath === fingerprint.resolvedHomePath,
770-
),
771-
),
772-
);
773-
if (marked.size === 0) return null;
774-
return {
775-
...summary,
776-
sources: summary.sources.map((source) =>
777-
marked.has(source) ? { ...source, refreshing: true } : source,
778-
),
779-
};
780-
};
781-
782729
const scanSummary = Effect.fn("UsageService.scanSummary")(function* (
783730
input: UsageSummaryInput,
784731
settings: ServerSettingsValue,
@@ -830,21 +777,23 @@ export const make = Effect.gen(function* () {
830777

831778
const retentionCutoffMs = startedAtMs - CACHE_RETENTION_DAYS * 24 * 60 * 60 * 1000;
832779

833-
const [, { scanned: scannedDirs, fromMemory, unread }] = yield* Effect.all(
780+
const awaitRefresh = input.awaitRefresh === true;
781+
const resolved = yield* resolveTranscriptDirs(settings, retentionCutoffMs, awaitRefresh).pipe(
782+
Effect.provideContext(readerContext),
783+
);
784+
// A window no complete scan has read would show partial totals, so every directory must cover it.
785+
const fromMemory =
786+
provisional &&
787+
resolved.dirs.every(
788+
(source) => (coveredSince.get(coverageKey(source)) ?? Infinity) <= windowStartMs,
789+
);
790+
const [, scannedDirs] = yield* Effect.all(
834791
[
835-
// Loaded rates skip the lock, which an exact read holds through a slow fetch.
836-
provisional && rates.size > 0 ? Effect.void : ensureRates(false, provisional),
837-
collectDirs(
838-
windowStartMs,
839-
settings,
840-
retentionCutoffMs,
841-
input.awaitRefresh === true,
842-
provisional,
843-
),
792+
fromMemory ? loadSavedRates : ensureRates(false),
793+
collectDirs(resolved, windowStartMs, settings, retentionCutoffMs, awaitRefresh, fromMemory),
844794
],
845795
{ concurrency: 2 },
846796
);
847-
if (provisional && !fromMemory) yield* ensureRates(false);
848797

849798
const aggregator = new UsageAggregator({
850799
timeZone: input.timeZone,
@@ -897,25 +846,16 @@ export const make = Effect.gen(function* () {
897846
});
898847
// Usage two transcripts share counts in the first one, so saved files keep the order of the last walk.
899848
for (const files of fromMemory ? [] : filesByDir) {
900-
const saved = files.flatMap((file) => (fileCache.has(file.path) ? [file.path] : []));
901-
if (saved.length < 2) continue;
902-
const members = new Set(saved);
903-
const expected = saved.values();
904-
if (
905-
fileCache
906-
.keys()
907-
.filter((filePath) => members.has(filePath))
908-
.every((filePath) => filePath === expected.next().value)
909-
)
910-
continue;
911-
// Reorder within the slots these files hold, so files this walk did not list keep their place.
849+
const walked = files.flatMap((file) => (fileCache.has(file.path) ? [file.path] : []));
850+
const members = new Set(walked);
851+
const saved = [...fileCache.keys()];
852+
let next = 0;
853+
// Only the slots these files hold change, so files this walk did not list keep their place.
854+
const ordered = saved.map((filePath) => (members.has(filePath) ? walked[next++]! : filePath));
855+
if (ordered.every((filePath, index) => filePath === saved[index])) continue;
912856
const entries = new Map(fileCache);
913-
const walked = saved.values();
914857
fileCache.clear();
915-
for (const filePath of entries.keys()) {
916-
const slot = members.has(filePath) ? walked.next().value! : filePath;
917-
fileCache.set(slot, entries.get(slot)!);
918-
}
858+
for (const filePath of ordered) fileCache.set(filePath, entries.get(filePath)!);
919859
cacheDirty = true;
920860
}
921861
const windowEndMs =
@@ -973,6 +913,7 @@ export const make = Effect.gen(function* () {
973913
// Distinct per directory. Buckets carry per-cell session counts, but a
974914
// session spans days and models, so clients total this figure instead.
975915
const sessionIds = new Set<string>();
916+
let counted = false;
976917

977918
for (const file of filesByDir[index] ?? []) {
978919
malformedRecords += file.malformedRecords;
@@ -1001,8 +942,9 @@ export const make = Effect.gen(function* () {
1001942
eventOccurrences.set(key, occurrence);
1002943
usageRecord = { ...record, dedupeKey: key + ":" + occurrence };
1003944
}
1004-
if (aggregator.add(usageRecord, dir) && record.sessionId.length > 0) {
1005-
sessionIds.add(record.sessionId);
945+
if (aggregator.add(usageRecord, dir)) {
946+
counted = true;
947+
if (record.sessionId.length > 0) sessionIds.add(record.sessionId);
1006948
}
1007949
}
1008950
}
@@ -1029,14 +971,18 @@ export const make = Effect.gen(function* () {
1029971
.filter(Boolean)
1030972
.join(" ")
1031973
: sourceMessage,
1032-
...(refreshing ? { refreshing } : {}),
974+
// A saved answer misses what changed since its scan, so the client must ask again. An
975+
// empty source stays unmarked: it would only flash a loading row.
976+
...(refreshing || (fromMemory && counted && transcriptFormats.has(provider))
977+
? { refreshing: true }
978+
: {}),
1033979
});
1034980
}
1035981

1036982
const pruned = pruneScanCache(fileCache, retentionCutoffMs);
1037983
if (pruned > 0) {
1038984
cacheDirty = true;
1039-
// Pruned files are gone from memory, so nothing before the cutoff is covered any more.
985+
// Pruned files are no longer saved, so nothing before the cutoff stays covered.
1040986
for (const [key, since] of coveredSince) {
1041987
if (since < retentionCutoffMs) coveredSince.set(key, retentionCutoffMs);
1042988
}
@@ -1046,7 +992,7 @@ export const make = Effect.gen(function* () {
1046992
const readAt = yield* DateTime.now;
1047993
const finishedAtMs = yield* Clock.currentTimeMillis;
1048994

1049-
const summary: UsageSummary = {
995+
const summary = {
1050996
contractVersion: USAGE_CONTRACT_VERSION,
1051997
readAt: DateTime.formatIso(readAt),
1052998
timeZone: input.timeZone,
@@ -1057,30 +1003,29 @@ export const make = Effect.gen(function* () {
10571003
pricing: pricing(),
10581004
scanDurationMs: Math.max(0, finishedAtMs - startedAtMs),
10591005
...(input.modelFilter === undefined ? {} : { modelFilter: input.modelFilter }),
1060-
};
1061-
return { summary, fromMemory, unread };
1006+
} satisfies UsageSummary;
1007+
return { summary, fromMemory };
10621008
});
10631009

10641010
const readWindow = Effect.fn("UsageService.readWindow")(function* (
10651011
input: UsageSummaryInput,
10661012
settings: ServerSettingsValue,
10671013
provisional: boolean,
10681014
) {
1069-
const scan = yield* scanSummary(input, settings, provisional);
1070-
if (!scan.fromMemory) return scan;
1071-
const saved = asSaved(scan.summary);
1072-
return saved === null
1073-
? yield* scanSummary(input, settings, false)
1074-
: { ...scan, summary: saved };
1015+
const { summary, fromMemory } = yield* scanSummary(input, settings, provisional);
1016+
// With nothing marked refreshing the client would keep a saved answer as final.
1017+
return fromMemory && !summary.sources.some((source) => source.refreshing === true)
1018+
? (yield* scanSummary(input, settings, false)).summary
1019+
: summary;
10751020
});
10761021

10771022
const inflightScans = new Map<string, Deferred.Deferred<UsageSummary, UsageReadError>>();
10781023

1079-
// The last exact answer per window and settings; a provisional read returns it without aggregating.
1080-
const answered = new Map<string, UsageSummary>();
1081-
const ANSWERED_WINDOWS = 32;
1082-
1083-
const windowKey = (input: UsageSummaryInput, settings: ServerSettingsValue): string =>
1024+
const scanKey = (
1025+
input: UsageSummaryInput,
1026+
settings: ServerSettingsValue,
1027+
provisional: boolean,
1028+
): string =>
10841029
JSON.stringify([
10851030
input.timeZone,
10861031
input.sinceDay,
@@ -1089,6 +1034,8 @@ export const make = Effect.gen(function* () {
10891034
input.sinceTime ?? null,
10901035
input.untilTime ?? null,
10911036
settings,
1037+
input.awaitRefresh === true,
1038+
provisional,
10921039
input.modelFilter ?? null,
10931040
]);
10941041

@@ -1099,12 +1046,8 @@ export const make = Effect.gen(function* () {
10991046
const modelFilter = canonicalModelFilter(requestedFilter);
11001047
const input = modelFilter === undefined ? unfiltered : { ...unfiltered, modelFilter };
11011048
const settings = yield* readSettings;
1102-
const window = windowKey(input, settings);
11031049
const provisional = input.provisional === true && input.awaitRefresh !== true;
1104-
const last = provisional ? answered.get(window) : undefined;
1105-
const saved = last === undefined ? null : asSaved(last);
1106-
if (saved !== null) return saved;
1107-
const key = `${window}\0${input.awaitRefresh === true}\0${provisional}`;
1050+
const key = scanKey(input, settings, provisional);
11081051
const deferred = yield* Effect.uninterruptible(
11091052
Effect.gen(function* () {
11101053
const existing = inflightScans.get(key);
@@ -1116,22 +1059,6 @@ export const make = Effect.gen(function* () {
11161059
// Detach so caller departure cannot cancel shared scans; finished scans warm the cache.
11171060
// Register writes before releasing waiters, but start writing after delivering their summary.
11181061
yield* readWindow(input, settings, provisional).pipe(
1119-
Effect.map(({ summary, unread }) => {
1120-
// An hourly window moves every minute, so its key never repeats. A scan that
1121-
// could not read every path is not kept: its answer may miss files.
1122-
if (
1123-
input.resolution === "hour" ||
1124-
unread ||
1125-
summary.sources.some((source) => source.refreshing === true)
1126-
)
1127-
return summary;
1128-
answered.delete(window);
1129-
answered.set(window, summary);
1130-
if (answered.size > ANSWERED_WINDOWS) {
1131-
answered.delete(answered.keys().next().value!);
1132-
}
1133-
return summary;
1134-
}),
11351062
Effect.onExit((exit) =>
11361063
Effect.sync(() => inflightScans.delete(key)).pipe(
11371064
Effect.andThen(Exit.isSuccess(exit) ? schedulePersist : Effect.void),

0 commit comments

Comments
 (0)