Skip to content

Commit c03f84b

Browse files
fix(server): stream the usage-scan cache to disk instead of one string (#34030)
1 parent c192c67 commit c03f84b

6 files changed

Lines changed: 108 additions & 38 deletions

File tree

‎apps/server/src/usage/UsageService.test.ts‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2587,7 +2587,7 @@ describe("UsageService", () => {
25872587
);
25882588

25892589
it.live.each([4, 5, 6, 7, 8, 10])(
2590-
"upgrades a v%s cache: reparses live Codex usage, keeps deleted rollouts and the old file",
2590+
"upgrades a v%s cache: reparses live Codex usage, keeps deleted rollouts, removes the old file",
25912591
(version) =>
25922592
Effect.gen(function* () {
25932593
const { home, settings } = yield* setup;
@@ -2636,7 +2636,7 @@ describe("UsageService", () => {
26362636
yield* first.readSummary(WINDOW);
26372637
yield* first.awaitPersisted;
26382638

2639-
const legacy = yield* Effect.promise(async () => {
2639+
yield* Effect.promise(async () => {
26402640
const livePath = await NodeFSP.realpath(live);
26412641
const document = decodeUnknownJsonString(await NodeFSP.readFile(cachePath, "utf8")) as {
26422642
files: Record<string, { r: unknown[][]; mr: number; cs: Record<string, unknown> }>;
@@ -2674,7 +2674,9 @@ describe("UsageService", () => {
26742674
}
26752675
}
26762676
await NodeFSP.rm(deleted);
2677-
return text;
2677+
await NodeFSP.mkdir(`${cachePath}.stale`);
2678+
await NodeFSP.utimes(`${cachePath}.stale`, 0, 0);
2679+
await NodeFSP.mkdir(`${cachePath}.active`);
26782680
});
26792681

26802682
const upgraded = yield* UsageService.make;
@@ -2686,9 +2688,11 @@ describe("UsageService", () => {
26862688
);
26872689
yield* upgraded.awaitPersisted;
26882690
assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v11.json");
2689-
assert.strictEqual(
2690-
yield* Effect.promise(() => NodeFSP.readFile(legacyPath, "utf8")),
2691-
legacy,
2691+
assert.deepStrictEqual(
2692+
(yield* Effect.promise(() => NodeFSP.readdir(stateDir)))
2693+
.filter((name) => name.startsWith("usage-scan-cache"))
2694+
.toSorted(),
2695+
[SCAN_CACHE_FILE_NAME, `${SCAN_CACHE_FILE_NAME}.active`],
26922696
);
26932697
}).pipe(
26942698
Effect.provide(

‎apps/server/src/usage/UsageService.ts‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ const MTIME_SLACK_MS = 36 * 60 * 60 * 1000;
7171
const MAX_HOURLY_WINDOW_MS = 24 * 60 * 60 * 1000;
7272

7373
const CACHE_RETENTION_DAYS = 92;
74+
const STALE_TEMP_DIRECTORY_MS = 60 * 60 * 1000;
7475

7576
const TRANSCRIPT_READ_CONCURRENCY = 4;
7677

@@ -172,7 +173,7 @@ export const make = Effect.gen(function* () {
172173

173174
const ratesCachePath = path.join(config.stateDir, "usage-model-rates.json");
174175
const scanCachePath = path.join(config.stateDir, SCAN_CACHE_FILE_NAME);
175-
const writeCacheFile = (filePath: string, contents: string) =>
176+
const writeCacheFile = (filePath: string, contents: string | Iterable<string>) =>
176177
writeFileStringAtomically({ filePath, contents }).pipe(
177178
Effect.provideService(FileSystem.FileSystem, fileSystem),
178179
Effect.provideService(Path.Path, path),
@@ -407,6 +408,32 @@ export const make = Effect.gen(function* () {
407408
// Serialize writes so concurrently finished window scans cannot overwrite newer snapshots.
408409
const persistLock = yield* Semaphore.make(1);
409410

411+
// The saved cache supersedes older versions. Temp folders an aborted write left behind are
412+
// removed once idle for an hour, so a write in progress from another server survives.
413+
let supersededFilesRemoved = false;
414+
const removeSupersededCacheFiles = Effect.gen(function* () {
415+
for (const fileName of LEGACY_SCAN_CACHE_FILE_NAMES) {
416+
yield* fileSystem.remove(path.join(config.stateDir, fileName)).pipe(Effect.ignore);
417+
}
418+
const tempPrefixes = [SCAN_CACHE_FILE_NAME, ...LEGACY_SCAN_CACHE_FILE_NAMES].map(
419+
(fileName) => `${fileName}.`,
420+
);
421+
const staleBeforeMs = (yield* Clock.currentTimeMillis) - STALE_TEMP_DIRECTORY_MS;
422+
for (const name of yield* fileSystem.readDirectory(config.stateDir)) {
423+
if (!tempPrefixes.some((prefix) => name.startsWith(prefix))) continue;
424+
const tempPath = path.join(config.stateDir, name);
425+
yield* fileSystem.stat(tempPath).pipe(
426+
Effect.flatMap((info) => {
427+
const mtimeMs = Option.getOrUndefined(info.mtime)?.getTime();
428+
return info.type === "Directory" && mtimeMs !== undefined && mtimeMs <= staleBeforeMs
429+
? fileSystem.remove(tempPath, { recursive: true })
430+
: Effect.void;
431+
}),
432+
Effect.ignore,
433+
);
434+
}
435+
}).pipe(Effect.ignore);
436+
410437
// Clear dirty before encoding to retain concurrent changes; restore it on failure for retry.
411438
const persistCaches = Effect.gen(function* () {
412439
if (cacheDirty) {
@@ -415,6 +442,13 @@ export const make = Effect.gen(function* () {
415442
writeScanCache(fileCache, { sources: Object.fromEntries(sourceCache) }),
416443
).pipe(
417444
Effect.flatMap((contents) => writeCacheFile(scanCachePath, contents)),
445+
Effect.andThen(
446+
Effect.suspend(() => {
447+
if (supersededFilesRemoved) return Effect.void;
448+
supersededFilesRemoved = true;
449+
return removeSupersededCacheFiles;
450+
}),
451+
),
418452
Effect.catchCause(() =>
419453
Effect.sync(() => {
420454
cacheDirty = true;

‎apps/server/src/usage/usageScanCache.test.ts‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ import type { UsageRecord } from "@t3tools/provider-core/server/usage";
1111
import { TEST_FORMAT_MAP } from "./usageTestFormats.ts";
1212
import { initialCodexScanState, parseCodexLine } from "../provider/Drivers/codexUsage.ts";
1313

14+
const parse = (parts: Iterable<string>) => JSON.parse([...parts].join(""));
15+
1416
function record(overrides: Partial<UsageRecord> = {}): UsageRecord {
1517
return {
1618
provider: "claude",
@@ -152,10 +154,7 @@ describe("scan cache round trip", () => {
152154
record({ model: "<synthetic>", dedupeKey: "msg_tail:" }),
153155
],
154156
});
155-
const restored = decodeScanCache(
156-
JSON.parse(makeScanCacheWriter()(cached, {})),
157-
TEST_FORMAT_MAP,
158-
);
157+
const restored = decodeScanCache(parse(makeScanCacheWriter()(cached, {})), TEST_FORMAT_MAP);
159158

160159
expect(restored.size).toBe(4);
161160
expect(restored).toEqual(original);
@@ -196,7 +195,7 @@ describe("scan cache round trip", () => {
196195
malformedRecords: 0,
197196
};
198197
original.set("/clean-codex.jsonl", cleanCodex);
199-
const encoded = JSON.parse(makeScanCacheWriter()(original, {}));
198+
const encoded = parse(makeScanCacheWriter()(original, {}));
200199
const entry = encoded.files["/gone.jsonl"]!;
201200
const legacy = {
202201
...encoded,
@@ -251,7 +250,7 @@ describe("scan cache round trip", () => {
251250

252251
it("re-parses a Claude transcript cached before advisor usage was counted", () => {
253252
const original = cacheWith([["/claude.jsonl", 100, [record()]]]);
254-
const v9 = { ...JSON.parse(makeScanCacheWriter()(original, {})), version: 9 };
253+
const v9 = { ...parse(makeScanCacheWriter()(original, {})), version: 9 };
255254

256255
const restored = decodeScanCache(JSON.parse(JSON.stringify(v9)), TEST_FORMAT_MAP);
257256
expect(restored.get("/claude.jsonl")).toEqual({
@@ -269,15 +268,15 @@ describe("scan cache round trip", () => {
269268
]);
270269
cache.set("/b.jsonl", { ...cache.get("/b.jsonl")!, malformedRecords: 2 });
271270
const sources = { "claude\u0000/projects": { dir: "/projects", volumeId: "1:2" } };
272-
expect(decodeScanCache(JSON.parse(write(cache, { sources })), TEST_FORMAT_MAP)).toEqual(cache);
271+
expect(decodeScanCache(parse(write(cache, { sources })), TEST_FORMAT_MAP)).toEqual(cache);
273272

274273
cache.set("/b.jsonl", {
275274
...cache.get("/b.jsonl")!,
276275
size: 30,
277276
malformedRecords: 4,
278277
records: [record({ sessionId: "session-c", model: "claude-opus-5-5", dedupeKey: "msg_3:" })],
279278
});
280-
const document = JSON.parse(write(cache, { sources }));
279+
const document = parse(write(cache, { sources }));
281280
expect(decodeScanCache(document, TEST_FORMAT_MAP)).toEqual(cache);
282281
expect(document.sources).toEqual(sources);
283282
});
@@ -291,9 +290,7 @@ describe("scan cache round trip", () => {
291290
});
292291

293292
it("skips malformed file entries but keeps good ones", () => {
294-
const encoded = JSON.parse(
295-
makeScanCacheWriter()(cacheWith([["/good.jsonl", 100, [record()]]]), {}),
296-
);
293+
const encoded = parse(makeScanCacheWriter()(cacheWith([["/good.jsonl", 100, [record()]]]), {}));
297294
const withJunk = {
298295
...encoded,
299296
files: { ...encoded.files, "/bad.jsonl": { s: "nope", m: 1, p: "claude", r: [] } },
@@ -304,7 +301,7 @@ describe("scan cache round trip", () => {
304301
});
305302

306303
it("drops the whole entry when any row is corrupt, forcing a cold re-parse", () => {
307-
const encoded = JSON.parse(
304+
const encoded = parse(
308305
makeScanCacheWriter()(
309306
cacheWith([["/a.jsonl", 100, [record(), record({ dedupeKey: "msg_2:" })]]]),
310307
{},

‎apps/server/src/usage/usageScanCache.ts‎

Lines changed: 12 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ const SPEED_COMPATIBLE_SINCE_VERSION = 4;
1515
/**
1616
* An older server sharing that directory cannot read a newer cache and would
1717
* replace it, dropping saved usage for deleted transcripts. Separate files keep
18-
* both.
18+
* the newer one; older files are removed once it is saved.
1919
*/
2020
export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v11.json";
2121
export const LEGACY_SCAN_CACHE_FILE_NAMES = [
@@ -134,32 +134,28 @@ function serializeFile(entry: CachedFile, tables: InternTables): SerializedFile
134134
};
135135
}
136136

137-
/** Memoizes immutable entries by identity; growing intern tables preserve indexes, retaining unused strings until restart. */
137+
/** Yields the document in pieces so large caches never become one string; growing intern tables preserve indexes, retaining unused strings until restart. */
138138
export function makeScanCacheWriter(): (
139139
cache: ScanCache,
140140
extra: Readonly<Record<string, unknown>>,
141-
) => string {
141+
) => Iterable<string> {
142142
const tables = makeInternTables();
143-
const fragments = new WeakMap<CachedFile, string>();
144-
return (cache, extra) => {
145-
const files: string[] = [];
146-
for (const [path, entry] of cache) {
147-
let fragment = fragments.get(entry);
148-
if (fragment === undefined) {
149-
fragment = JSON.stringify(serializeFile(entry, tables));
150-
fragments.set(entry, fragment);
151-
}
152-
files.push(`${JSON.stringify(path)}:${fragment}`);
143+
function* encode(entries: readonly (readonly [string, CachedFile])[], extra: object) {
144+
yield '{"files":{';
145+
for (const [index, [path, entry]] of entries.entries()) {
146+
yield `${index === 0 ? "" : ","}${JSON.stringify(path)}:${JSON.stringify(serializeFile(entry, tables))}`;
153147
}
154148
// Encoded after the files, which may have added to the intern tables.
155-
const head = JSON.stringify({
149+
const tail = JSON.stringify({
156150
...extra,
157151
version: USAGE_SCAN_CACHE_VERSION,
158152
models: tables.models,
159153
sessions: tables.sessions,
160154
});
161-
return `${head.slice(0, -1)},"files":{${files.join(",")}}}`;
162-
};
155+
yield `},${tail.slice(1)}`;
156+
}
157+
// Snapshot now: the encoding runs later, while the file is written.
158+
return (cache, extra) => encode([...cache], extra);
163159
}
164160

165161
function isRecordArray(value: unknown): value is readonly unknown[] {

‎packages/shared/src/atomicWrite.test.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,22 @@ it.layer(NodeServices.layer)("writeFileStringAtomically", (it) => {
4444
}),
4545
);
4646

47+
it.effect("streams iterable contents across write batches", () =>
48+
Effect.gen(function* () {
49+
const fs = yield* FileSystem.FileSystem;
50+
const path = yield* Path.Path;
51+
const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-atomic-write-" });
52+
const filePath = path.join(root, "cache.json");
53+
yield* fs.writeFileString(filePath, "a much longer previous document");
54+
const chunks = ["é".repeat(700_000), "🙂".repeat(400_000), "end"];
55+
56+
yield* writeFileStringAtomically({ filePath, contents: chunks.values() });
57+
58+
assert.strictEqual(yield* fs.readFileString(filePath), chunks.join(""));
59+
assert.deepStrictEqual(yield* fs.readDirectory(root), ["cache.json"]);
60+
}),
61+
);
62+
4763
it.effect("keeps a symlinked file linked and rewrites its destination", () =>
4864
Effect.gen(function* () {
4965
const fs = yield* FileSystem.FileSystem;

‎packages/shared/src/atomicWrite.ts‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,12 @@ export const isWindowsRenameLock = (error: PlatformError.PlatformError) => {
1616
// Bound denial delays to about two seconds because settings saves hold their write lock.
1717
const LOCKED_RENAME_RETRIES = 20;
1818
const LOCKED_RENAME_RETRY_DELAY = "100 millis";
19+
const CHUNK_WRITE_LENGTH = 1 << 20;
1920

20-
/** Resolves symlink targets before renaming so writes preserve the link. */
21+
/** Resolves symlink targets before renaming so writes preserve the link. Iterable contents stream in bounded batches. */
2122
export const writeFileStringAtomically = (input: {
2223
readonly filePath: string;
23-
readonly contents: string;
24+
readonly contents: string | Iterable<string>;
2425
}) =>
2526
Effect.scoped(
2627
Effect.gen(function* () {
@@ -41,7 +42,29 @@ export const writeFileStringAtomically = (input: {
4142
);
4243
const tempPath = path.join(tempDirectory, "contents.tmp");
4344

44-
yield* fs.writeFileString(tempPath, input.contents);
45+
const contents = input.contents;
46+
if (typeof contents === "string") {
47+
yield* fs.writeFileString(tempPath, contents);
48+
} else {
49+
// Closed before the rename, which Windows refuses for open files.
50+
yield* Effect.scoped(
51+
Effect.gen(function* () {
52+
const file = yield* fs.open(tempPath, { flag: "w" });
53+
const encoder = new TextEncoder();
54+
let batch: string[] = [];
55+
let batchLength = 0;
56+
for (const chunk of contents) {
57+
batch.push(chunk);
58+
batchLength += chunk.length;
59+
if (batchLength < CHUNK_WRITE_LENGTH) continue;
60+
yield* file.writeAll(encoder.encode(batch.join("")));
61+
batch = [];
62+
batchLength = 0;
63+
}
64+
if (batchLength > 0) yield* file.writeAll(encoder.encode(batch.join("")));
65+
}),
66+
);
67+
}
4568
if (platform !== "win32") return yield* fs.rename(tempPath, targetPath);
4669

4770
// Stop retrying if another save changes the target; read-only targets also return permanent EPERM.

0 commit comments

Comments
 (0)