Skip to content

Commit bc378da

Browse files
committed
perf(server): reuse Find documents across query changes
1 parent d8f3fef commit bc378da

2 files changed

Lines changed: 175 additions & 30 deletions

File tree

‎apps/server/src/orchestration-v2/ThreadFind.test.ts‎

Lines changed: 83 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import * as Layer from "effect/Layer";
2121
import * as SqlClient from "effect/sql/SqlClient";
2222
import * as Stream from "effect/Stream";
2323
import * as TestClock from "effect/testing/TestClock";
24+
import * as Tracer from "effect/Tracer";
2425
import { vi } from "vite-plus/test";
2526

2627
import * as SqlitePersistence from "../persistence/Sqlite.ts";
@@ -204,14 +205,55 @@ const setup = Effect.gen(function* () {
204205
return yield* ProjectionStore.ProjectionStoreV2;
205206
});
206207

208+
const tracePayloadLoads = () => {
209+
let batches = 0;
210+
const tracer = Tracer.make({
211+
span(options) {
212+
if (options.name === "ThreadFind.load") batches++;
213+
return new Tracer.NativeSpan(options);
214+
},
215+
});
216+
return { tracer, batches: () => batches };
217+
};
218+
207219
describe("V2 thread find", () => {
220+
it.effect.each(["rendered text", "source text"] as const)(
221+
"does not retain rendered snapshots over the %s budget",
222+
(limit) =>
223+
Effect.gen(function* () {
224+
const projection = yield* setup;
225+
yield* putItems(
226+
Array.from({ length: 65 }, (_, i) =>
227+
item(
228+
`message:${i}`,
229+
i,
230+
limit === "rendered text"
231+
? "needle ".repeat(2500)
232+
: `needle ![image](data:image/png;base64,${"a".repeat(35000)})`,
233+
),
234+
),
235+
);
236+
const loads = tracePayloadLoads();
237+
const first = yield* projection
238+
.searchThread({ threadId, query: "needle" })
239+
.pipe(Effect.withTracer(loads.tracer));
240+
const initialBatches = loads.batches();
241+
const second = yield* projection
242+
.searchThread({ threadId, query: "need" })
243+
.pipe(Effect.withTracer(loads.tracer));
244+
assert.equal(second.totalMatches, first.totalMatches);
245+
assert.equal(loads.batches(), initialBatches * 2);
246+
}).pipe(Effect.provide(layerTest)),
247+
);
248+
208249
it.effect("finishes one snapshot while later searches include newly inserted matches", () =>
209250
Effect.gen(function* () {
210251
const projection = yield* setup;
211252
yield* putItems([
212253
item("selected", 1, "snapshot needle"),
213254
item("last", 2, "snapshot last needle"),
214255
]);
256+
yield* projection.searchThread({ threadId, query: "snapshot" });
215257
const frames = yield* projection.searchThreadStream({ threadId, query: "needle" }).pipe(
216258
Stream.tap((frame) =>
217259
frame.complete === false
@@ -391,6 +433,9 @@ describe("V2 thread find", () => {
391433
const result = yield* projection.searchThread({ threadId, query: "empty" });
392434
assert.equal(result.totalMatches, 1);
393435
assert.equal(result.match?.entryId, "shown");
436+
const changed = yield* projection.searchThread({ threadId, query: "response" });
437+
assert.equal(changed.totalMatches, 1);
438+
assert.equal(changed.match?.entryId, "shown");
394439
}).pipe(Effect.provide(layerTest)),
395440
);
396441

@@ -418,6 +463,28 @@ describe("V2 thread find", () => {
418463
}).pipe(Effect.provide(layerTest)),
419464
);
420465

466+
it.effect("refreshes file labels when the project cwd changes without a thread event", () =>
467+
Effect.gen(function* () {
468+
const projection = yield* setup;
469+
const sql = yield* SqlClient.SqlClient;
470+
yield* putItems([item("paths", 1, "[local](src/main.ts) [other](/elsewhere/src/main.ts)")]);
471+
assert.equal(
472+
(yield* projection.searchThread({ threadId, query: "project:find/src" })).totalMatches,
473+
1,
474+
);
475+
yield* sql`UPDATE projection_projects SET workspace_root = '/elsewhere'
476+
WHERE project_id = ${projectId}`;
477+
assert.equal(
478+
(yield* projection.searchThread({ threadId, query: "project:find" })).totalMatches,
479+
0,
480+
);
481+
assert.equal(
482+
(yield* projection.searchThread({ threadId, query: "main.ts" })).totalMatches,
483+
2,
484+
);
485+
}).pipe(Effect.provide(layerTest)),
486+
);
487+
421488
it.effect("never mixes message versions when the searched thread changes during a scan", () =>
422489
Effect.gen(function* () {
423490
const projection = yield* setup;
@@ -479,7 +546,17 @@ describe("V2 thread find", () => {
479546
item(`message:${i}`, i + 1, i === 8 || i === 510 ? "sentinel" : "other message"),
480547
);
481548
yield* putItems(items);
482-
const first = yield* projection.searchThread({ threadId, query: "sentinel" });
549+
const loads = tracePayloadLoads();
550+
const first = yield* projection
551+
.searchThread({ threadId, query: "sentinel" })
552+
.pipe(Effect.withTracer(loads.tracer));
553+
assert.equal(loads.batches(), 5);
554+
const changed = yield* projection
555+
.searchThread({ threadId, query: "other message" })
556+
.pipe(Effect.withTracer(loads.tracer));
557+
assert.equal(changed.totalMatches, 518);
558+
assert.equal(changed.match?.entryId, "message:0");
559+
assert.equal(loads.batches(), 5);
483560
const last = yield* projection.searchThread({ threadId, query: "sentinel", index: 1 });
484561
assert.equal(first.totalMatches, 2);
485562
assert.equal(first.match?.entryId, "message:8");
@@ -530,6 +607,7 @@ describe("V2 thread find", () => {
530607
if (original.type !== "assistant_message") return;
531608
yield* putItems([{ ...original, text: "needle needle", streaming: true }], "streaming");
532609
assert.equal((yield* projection.searchThread({ threadId, query: "needle" })).totalMatches, 2);
610+
assert.equal((yield* projection.searchThread({ threadId, query: "need" })).totalMatches, 2);
533611
const updated = run(threadId, runId, 1, "rolled_back");
534612
yield* commit([{ ...updated, type: "run.updated" }]);
535613
assert.equal((yield* projection.searchThread({ threadId, query: "needle" })).totalMatches, 0);
@@ -582,6 +660,9 @@ describe("V2 thread find", () => {
582660
const result = yield* projection.searchThread({ threadId, query: "needle" });
583661
assert.equal(result.totalMatches, 0);
584662
assert.isNull(result.match);
663+
assert.equal((yield* projection.searchThread({ threadId, query: "need" })).totalMatches, 0);
664+
yield* putItems([{ ...question, questionAnswer: undefined }], "unfolded");
665+
assert.equal((yield* projection.searchThread({ threadId, query: "need" })).totalMatches, 1);
585666
}).pipe(Effect.provide(layerTest)),
586667
);
587668

@@ -667,6 +748,7 @@ describe("V2 thread find", () => {
667748
(yield* projection.searchThread({ threadId, query: "needle", index: 1 })).match?.entryId,
668749
"local",
669750
);
751+
assert.equal((yield* projection.searchThread({ threadId, query: "need" })).totalMatches, 2);
670752
const snapshot = yield* projection.getThreadSnapshot(threadId);
671753
const history = yield* projection.getThreadHistoryPage(
672754
threadId,

‎apps/server/src/orchestration-v2/ThreadFind.ts‎

Lines changed: 92 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,10 @@ interface FindDocument extends FindRow {
4242
readonly runId: OrchestrationV2TurnItem["runId"];
4343
readonly count: number;
4444
}
45+
interface RenderedDocument extends Omit<FindDocument, "count"> {
46+
readonly segments: readonly string[];
47+
readonly sourceLength: number;
48+
}
4549
type CompletedScans = Map<
4650
string,
4751
{
@@ -50,6 +54,17 @@ type CompletedScans = Map<
5054
sourceThreadIds: readonly ThreadId[];
5155
}
5256
>;
57+
// Segments may retain slices of their source text, so the budget includes both.
58+
interface RenderedSnapshot {
59+
current:
60+
| {
61+
key: string;
62+
sequence: number;
63+
documents: readonly RenderedDocument[];
64+
sourceThreadIds: readonly ThreadId[];
65+
}
66+
| undefined;
67+
}
5368
class TextKey extends Data.Class<{
5469
text: string;
5570
role: "user" | "assistant" | "plan";
@@ -190,7 +205,7 @@ export const makeThreadFind = Effect.fn("makeThreadFind")(function* (
190205
Schema.Struct({
191206
threadId: ThreadId,
192207
cwd: Schema.optional(Schema.String),
193-
query: Schema.String,
208+
query: Schema.optional(Schema.String),
194209
skills: OrchestrationV2SearchThreadInput.fields.skills,
195210
}),
196211
),
@@ -262,6 +277,7 @@ export const makeThreadFind = Effect.fn("makeThreadFind")(function* (
262277
// Bounded separately because each entry can hold a large body; typing reuses the parse.
263278
largeText: makeTextCache(32),
264279
scans: Effect.sync((): CompletedScans => new Map()),
280+
rendered: Effect.sync((): RenderedSnapshot => ({ current: undefined })),
265281
}),
266282
idleTimeToLive: "1 minute",
267283
});
@@ -288,7 +304,11 @@ export const makeThreadFind = Effect.fn("makeThreadFind")(function* (
288304
return revision[0]?.sequence ?? 0;
289305
});
290306
const snapshot = Effect.fn("ThreadFind.snapshot")(
291-
function* (input: OrchestrationV2SearchThreadInput, scans: CompletedScans) {
307+
function* (
308+
input: OrchestrationV2SearchThreadInput,
309+
scans: CompletedScans,
310+
renderedSnapshot: RenderedSnapshot,
311+
) {
292312
const { threadId } = input;
293313
const active = yield* sql<{ cwd: string | null }>`
294314
SELECT COALESCE(json_extract(t.payload_json, '$.worktreePath'), p.workspace_root) AS cwd
@@ -310,6 +330,25 @@ export const makeThreadFind = Effect.fn("makeThreadFind")(function* (
310330
scans.set(cacheKey, cached);
311331
return { result: resultForSelection(selected, cached.documents, cached.sequence) };
312332
}
333+
const documentKey = yield* encodeCacheKey({
334+
threadId,
335+
cwd,
336+
skills: input.skills ?? [],
337+
});
338+
const rendered = renderedSnapshot.current;
339+
if (
340+
rendered?.key === documentKey &&
341+
rendered.sequence === (yield* readRevision(rendered.sourceThreadIds))
342+
) {
343+
return {
344+
rendered: rendered.documents,
345+
cwd,
346+
sequence: rendered.sequence,
347+
cacheKey,
348+
documentKey,
349+
sourceThreadIds: rendered.sourceThreadIds,
350+
};
351+
}
313352
const index = yield* readIndex(threadId);
314353
const sourceThreadIds = [...new Set([threadId, ...index.map((row) => row.sourceThreadId)])];
315354
const sequence = yield* readRevision(sourceThreadIds);
@@ -346,7 +385,7 @@ export const makeThreadFind = Effect.fn("makeThreadFind")(function* (
346385
emptyReplyRunIds.has(row.item.runId),
347386
),
348387
);
349-
return { payloads, toolPayloads, cwd, sequence, cacheKey, sourceThreadIds };
388+
return { payloads, toolPayloads, cwd, sequence, cacheKey, documentKey, sourceThreadIds };
350389
},
351390
(effect) => sql.withTransaction(effect).pipe(NodeSqliteClient.readOnly),
352391
(effect, input) =>
@@ -361,11 +400,13 @@ export const makeThreadFind = Effect.fn("makeThreadFind")(function* (
361400
const searchThreadStream = (input: OrchestrationV2SearchThreadInput) =>
362401
Stream.unwrap(
363402
Effect.gen(function* () {
364-
const { scans, ...textCache } = yield* RcRef.get(findCaches);
403+
const { scans, rendered: renderedSnapshot, ...textCache } = yield* RcRef.get(findCaches);
365404
// Release SQLite before decoding and parsing. Both frames describe one snapshot.
366-
const data = yield* snapshot(input, scans);
405+
const data = yield* snapshot(input, scans, renderedSnapshot);
367406
if (data.result) return Stream.succeed(data.result);
368-
const { payloads, toolPayloads, cwd, sequence, cacheKey, sourceThreadIds } = data;
407+
const { cwd, sequence, cacheKey, documentKey, sourceThreadIds } = data;
408+
const payloads = data.payloads ?? [];
409+
const toolPayloads = data.toolPayloads ?? [];
369410
const decode = (row: FindPayload) =>
370411
decodeItem(row.payload).pipe(
371412
Effect.map((item) => ({ ...row, item })),
@@ -392,40 +433,62 @@ export const makeThreadFind = Effect.fn("makeThreadFind")(function* (
392433
)
393434
embeddedResultRunIds.add(item.runId);
394435
}
395-
const rows = payloads.filter(
396-
(row) =>
397-
row.type !== "user_input_request" &&
398-
!(row.type === "user_message" && foldedAnswerRunIds.get(row.entryId) === row.runId),
399-
);
400-
const documents = new Map<number, FindDocument>();
401-
const count = Effect.fnUntraced(function* (row: FindPayload) {
436+
const rows =
437+
data.rendered ??
438+
payloads.filter(
439+
(row) =>
440+
row.type !== "user_input_request" &&
441+
!(row.type === "user_message" && foldedAnswerRunIds.get(row.entryId) === row.runId),
442+
);
443+
const documents = new Map<number, FindDocument & RenderedDocument>();
444+
const count = Effect.fnUntraced(function* (row: FindPayload | RenderedDocument) {
402445
const cached = documents.get(row.position);
403446
if (cached) return cached;
404-
const decoded = yield* decode(row);
405-
if (
406-
decoded.item.type === "assistant_message" &&
407-
decoded.item.text === "" &&
408-
embeddedResultRunIds.has(decoded.item.runId)
409-
) {
410-
const document = documentFor(decoded, 0);
411-
documents.set(row.position, document);
412-
return document;
447+
let rendered: RenderedDocument;
448+
if ("segments" in row) {
449+
rendered = row;
450+
} else {
451+
const decoded = yield* decode(row);
452+
const key = textKey(decoded.item, cwd, input.skills ?? []);
453+
const segments =
454+
decoded.item.type === "assistant_message" &&
455+
decoded.item.text === "" &&
456+
embeddedResultRunIds.has(decoded.item.runId)
457+
? []
458+
: yield* Cache.get(
459+
key.text.length <= 32_768 ? textCache.text : textCache.largeText,
460+
key,
461+
);
462+
rendered = { ...documentFor(decoded, 0), segments, sourceLength: key.text.length };
413463
}
414-
const key = textKey(decoded.item, cwd, input.skills ?? []);
415-
const segments = yield* Cache.get(
416-
key.text.length <= 32_768 ? textCache.text : textCache.largeText,
417-
key,
418-
);
419-
const document = documentFor(decoded, countSegments(segments, input.query));
464+
const document = {
465+
...rendered,
466+
count: countSegments(rendered.segments, input.query),
467+
};
420468
documents.set(row.position, document);
421469
yield* Effect.yieldNow;
422470
return document;
423471
});
424472
const finish = Effect.gen(function* () {
425473
for (const row of rows) yield* count(row);
426474
const ordered = rows.map((row) => documents.get(row.position)!);
475+
renderedSnapshot.current =
476+
ordered.length <= 8192 &&
477+
ordered.reduce(
478+
(size, doc) =>
479+
size + doc.sourceLength + doc.segments.reduce((sum, text) => sum + text.length, 0),
480+
0,
481+
) <= 2_097_152
482+
? { key: documentKey, sequence, documents: ordered, sourceThreadIds }
483+
: undefined;
427484
scans.delete(cacheKey);
428-
scans.set(cacheKey, { sequence, documents: ordered, sourceThreadIds });
485+
scans.set(cacheKey, {
486+
sequence,
487+
documents: ordered.map(
488+
({ segments: _segments, sourceLength: _sourceLength, ...document }) => document,
489+
),
490+
sourceThreadIds,
491+
});
429492
if (scans.size > 8) {
430493
const oldest = scans.keys().next().value;
431494
if (oldest !== undefined) scans.delete(oldest);

0 commit comments

Comments
 (0)