Skip to content

Commit 9ec472c

Browse files
tris203t3dotggclaude
authored
perf(usage): OpenCode legacy walk stats entries before checking for symlinks (pingdotgg#17638)
Co-authored-by: Theo Browne <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent febe1b2 commit 9ec472c

2 files changed

Lines changed: 161 additions & 30 deletions

File tree

‎packages/provider-opencode/src/server/usage.test.ts‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,5 +182,95 @@ describe("readOpenCodeUsage", () => {
182182
);
183183
}).pipe(Effect.provide(NodeServices.layer)),
184184
);
185+
186+
it.effect("never follows symlinks in the legacy OpenCode store", () =>
187+
Effect.gen(function* () {
188+
const fileSystem = yield* FileSystem.FileSystem;
189+
const path = yield* Path.Path;
190+
const dir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "usage-reader-test-" });
191+
const outside = yield* fileSystem.makeTempDirectoryScoped({ prefix: "usage-reader-test-" });
192+
const sinceMs = 1780000000000;
193+
const store = path.join(dir, "storage", "message");
194+
const session = path.join(store, "session-1");
195+
yield* fileSystem.makeDirectory(session, { recursive: true });
196+
const write = Effect.fn(function* (directory: string, id: string, mtimeMs: number) {
197+
const file = path.join(directory, `${id}.json`);
198+
yield* fileSystem.writeFileString(
199+
file,
200+
JSON.stringify({
201+
id,
202+
sessionID: "session-1",
203+
role: "assistant",
204+
modelID: "claude-sonnet-4-5",
205+
time: { created: sinceMs + 1000 },
206+
tokens: { input: 100, output: 20 },
207+
}),
208+
);
209+
yield* fileSystem.utimes(file, mtimeMs / 1000, mtimeMs / 1000);
210+
return file;
211+
});
212+
const target = yield* write(outside, "msg-linked", sinceMs + 60_000);
213+
const stale = yield* write(outside, "msg-stale", sinceMs - 60_000);
214+
yield* fileSystem.symlink(target, path.join(session, "msg-linked.json"));
215+
yield* fileSystem.symlink(stale, path.join(session, "msg-stale.json"));
216+
yield* fileSystem.symlink(outside, path.join(store, "session-linked"));
217+
yield* fileSystem.symlink(store, path.join(session, "cycle"));
218+
yield* fileSystem.symlink(path.join(outside, "gone.json"), path.join(session, "gone.json"));
219+
yield* fileSystem.symlink(path.join(session, "loop.json"), path.join(session, "loop.json"));
220+
221+
// Links alone are not a store, even when one points at a stale message.
222+
const linksOnly = yield* readOpenCodeUsage(dir, sinceMs);
223+
assert.deepStrictEqual(linksOnly, { files: [], missing: true, error: false });
224+
225+
const real = yield* write(session, "msg-real", sinceMs + 60_000);
226+
const result = yield* readOpenCodeUsage(dir, sinceMs);
227+
assert.isFalse(result.missing);
228+
assert.isFalse(result.error);
229+
assert.deepStrictEqual(
230+
result.files.map((file) => file.path),
231+
[real],
232+
);
233+
assert.deepStrictEqual(
234+
result.files.flatMap((file) => file.records).map((record) => record.dedupeKey),
235+
["opencode:msg-real"],
236+
);
237+
}).pipe(Effect.provide(NodeServices.layer)),
238+
);
239+
240+
it.effect("reads a large legacy OpenCode session in directory order", () =>
241+
Effect.gen(function* () {
242+
const fileSystem = yield* FileSystem.FileSystem;
243+
const path = yield* Path.Path;
244+
const dir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "usage-reader-test-" });
245+
const session = path.join(dir, "storage", "message", "session-1");
246+
yield* fileSystem.makeDirectory(session, { recursive: true });
247+
// More entries than the walk stats at once.
248+
yield* Effect.forEach(
249+
Array.from({ length: 600 }, (_, index) => `msg-${index}`),
250+
(id) =>
251+
fileSystem.writeFileString(
252+
path.join(session, `${id}.json`),
253+
JSON.stringify({
254+
id,
255+
sessionID: "session-1",
256+
role: "assistant",
257+
modelID: "claude-sonnet-4-5",
258+
time: { created: 1780000000000 },
259+
tokens: { input: 100, output: 20 },
260+
}),
261+
),
262+
{ concurrency: 16, discard: true },
263+
);
264+
const names = yield* fileSystem.readDirectory(session);
265+
266+
const result = yield* readOpenCodeUsage(dir, 0);
267+
assert.isFalse(result.error);
268+
assert.deepStrictEqual(
269+
result.files.map((file) => file.path),
270+
names.map((name) => path.join(session, name)),
271+
);
272+
assert.strictEqual(result.files.flatMap((file) => file.records).length, 600);
273+
}).pipe(Effect.provide(NodeServices.layer)),
274+
);
185275
});
186276
import * as NodeSqlite from "node:sqlite";

‎packages/provider-opencode/src/server/usage.ts‎

Lines changed: 71 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ import {
1414
import * as HostProcess from "@t3tools/shared/HostProcess";
1515
import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
1616
import * as Effect from "effect/Effect";
17-
import * as Exit from "effect/Exit";
1817
import * as FileSystem from "effect/FileSystem";
1918
import * as Option from "effect/Option";
2019
import * as Path from "effect/Path";
@@ -94,14 +93,29 @@ export interface OpenCodeUsageReadResult {
9493
const isNotFound = (cause: PlatformError.PlatformError) => cause.reason._tag === "NotFound";
9594
const MESSAGE_PAGE_SIZE = 64;
9695

96+
/** Whether the path is itself a symlink, without following it. */
97+
const isSymbolicLink = Effect.fnUntraced(function* (path: string) {
98+
const fileSystem = yield* FileSystem.FileSystem;
99+
return yield* Effect.isSuccess(fileSystem.readLink(path));
100+
});
101+
97102
/** Stat of a regular file or directory; a symlink reports as itself and is never followed. */
98103
const entryInfo = Effect.fn("entryInfo")(function* (path: string) {
99104
const fileSystem = yield* FileSystem.FileSystem;
100-
const link = yield* Effect.exit(fileSystem.readLink(path));
101-
if (Exit.isSuccess(link)) return { type: "SymbolicLink" as const, mtime: Option.none<Date>() };
105+
if (yield* isSymbolicLink(path))
106+
return { type: "SymbolicLink" as const, mtime: Option.none<Date>() };
102107
return yield* fileSystem.stat(path);
103108
});
104109

110+
/**
111+
* Stats run in the libuv thread pool, so a few at a time cut the walk's wall
112+
* time without lengthening any event-loop turn.
113+
*/
114+
const LEGACY_STAT_CONCURRENCY = 4;
115+
116+
/** Entries stat'd before any is processed, so one huge directory is not held in memory at once. */
117+
const LEGACY_STAT_BATCH = 256;
118+
105119
/** Reads current SQLite and pre-migration JSON stores without modifying either. */
106120
export const readOpenCodeUsage = Effect.fn("readOpenCodeUsage")(function* (
107121
root: string,
@@ -234,34 +248,61 @@ export const readOpenCodeUsage = Effect.fn("readOpenCodeUsage")(function* (
234248
while (directories.length > 0) {
235249
const directory = directories.pop()!;
236250
yield* Effect.gen(function* () {
237-
for (const name of yield* fileSystem.readDirectory(directory)) {
238-
const entry = path.join(directory, name);
239-
const info = yield* entryInfo(entry).pipe(
240-
Effect.catchTags({
241-
PlatformError: (cause) =>
242-
isNotFound(cause) ? Effect.succeed(null) : Effect.fail(cause),
243-
}),
251+
const names = yield* fileSystem.readDirectory(directory);
252+
for (let start = 0; start < names.length; start += LEGACY_STAT_BATCH) {
253+
const batch = names.slice(start, start + LEGACY_STAT_BATCH);
254+
const entries = batch.map((name) => path.join(directory, name));
255+
// `stat` follows symlinks and `readLink` fails, expensively, for
256+
// everything else. Stat a batch, then ask whether an entry is a symlink
257+
// only where the answer changes the result.
258+
const infos = yield* Effect.forEach(
259+
entries,
260+
(entry) =>
261+
fileSystem.stat(entry).pipe(
262+
Effect.catchTags({
263+
// A symlink whose target cannot be stat'd is skipped, not an error.
264+
PlatformError: (cause) =>
265+
isNotFound(cause)
266+
? Effect.succeed(null)
267+
: isSymbolicLink(entry).pipe(
268+
Effect.flatMap((link) =>
269+
link ? Effect.succeed(null) : Effect.fail(cause),
270+
),
271+
),
272+
}),
273+
Effect.exit,
274+
),
275+
{ concurrency: LEGACY_STAT_CONCURRENCY },
244276
);
245-
if (info?.type === "Directory") {
246-
directories.push(entry);
247-
} else if (info?.type === "File" && name.endsWith(".json")) {
248-
found = true;
249-
const id = name.slice(0, -5);
250-
if (seen.has(`opencode:${id}`)) continue;
251-
// A message cannot be created after its file was last written, so a
252-
// file untouched since the window opened holds nothing in range.
253-
if (Option.exists(info.mtime, (mtime) => mtime.getTime() < sinceMs)) continue;
254-
const file = { path: entry, records: [] as UsageRecord[] };
255-
files.push(file);
256-
yield* fileSystem.readFileString(entry).pipe(
257-
Effect.map((source) => append(file.records, parseOpenCodeMessage(source, { id }))),
258-
Effect.catchTags({
259-
PlatformError: (cause) => {
260-
if (!isNotFound(cause)) error = true;
261-
return Effect.void;
262-
},
263-
}),
264-
);
277+
for (const [index, name] of batch.entries()) {
278+
const entry = entries[index]!;
279+
const info = yield* infos[index]!;
280+
if (info?.type === "Directory") {
281+
if (!(yield* isSymbolicLink(entry))) directories.push(entry);
282+
} else if (info?.type === "File" && name.endsWith(".json")) {
283+
const id = name.slice(0, -5);
284+
// A message cannot be created after its file was last written, so a
285+
// file untouched since the window opened holds nothing in range.
286+
const skip =
287+
seen.has(`opencode:${id}`) ||
288+
Option.exists(info.mtime, (mtime) => mtime.getTime() < sinceMs);
289+
// A skipped file only matters as proof that the store exists.
290+
if (found && skip) continue;
291+
if (yield* isSymbolicLink(entry)) continue;
292+
found = true;
293+
if (skip) continue;
294+
const file = { path: entry, records: [] as UsageRecord[] };
295+
files.push(file);
296+
yield* fileSystem.readFileString(entry).pipe(
297+
Effect.map((source) => append(file.records, parseOpenCodeMessage(source, { id }))),
298+
Effect.catchTags({
299+
PlatformError: (cause) => {
300+
if (!isNotFound(cause)) error = true;
301+
return Effect.void;
302+
},
303+
}),
304+
);
305+
}
265306
}
266307
}
267308
}).pipe(

0 commit comments

Comments
 (0)