Skip to content

Commit d7c7247

Browse files
committed
perf(server): read terminal successful task replies without scanning middle pages
1 parent e8adb99 commit d7c7247

2 files changed

Lines changed: 170 additions & 1 deletion

File tree

‎apps/server/src/peer/RemoteDelegation.test.ts‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ interface BoxThread {
9797
readonly timeline: Ref.Ref<ReadonlyArray<TimelineEntry> | null>;
9898
readonly pageSize: Ref.Ref<number>;
9999
readonly holdRefill: Ref.Ref<boolean>;
100+
readonly beforeTail: Ref.Ref<Effect.Effect<void>>;
100101
readonly launched: Ref.Ref<ThreadId | null>;
101102
/** The link session that launched it, as the box stamps it. */
102103
readonly linkOrigin: Ref.Ref<{ readonly sessionId: string; readonly label: string } | null>;
@@ -148,6 +149,7 @@ const makeBoxThread = Effect.gen(function* () {
148149
timeline: yield* Ref.make<ReadonlyArray<TimelineEntry> | null>(null),
149150
pageSize: yield* Ref.make(1),
150151
holdRefill: yield* Ref.make(false),
152+
beforeTail: yield* Ref.make<Effect.Effect<void>>(Effect.void),
151153
launched: yield* Ref.make<ThreadId | null>(null),
152154
linkOrigin: yield* Ref.make<{ readonly sessionId: string; readonly label: string } | null>(
153155
null,
@@ -317,6 +319,9 @@ const boxToolkitLayer = (thread: BoxThread) => {
317319
}),
318320
getTimelinePage: (threadId, options) =>
319321
Effect.gen(function* () {
322+
if (options.afterPosition === 299 && options.itemId === undefined) {
323+
yield* Ref.getAndSet(thread.beforeTail, Effect.void).pipe(Effect.flatten);
324+
}
320325
if (
321326
(yield* Ref.get(thread.holdRead)) ||
322327
(options.itemId !== undefined && (yield* Ref.get(thread.holdRefill)))
@@ -1530,6 +1535,143 @@ it.effect.each(["tail-error", "middle-error", "reply-fallback", "cancel-refill"]
15301535
}),
15311536
);
15321537

1538+
it.effect.each([
1539+
"terminal",
1540+
"long",
1541+
"later-run",
1542+
"sparse",
1543+
"growing",
1544+
"shrinking",
1545+
"cancel",
1546+
"no-run",
1547+
"cancelled",
1548+
"interrupted",
1549+
] as const)("reads a successful task with an aligned tail probe (%s)", (scenario) =>
1550+
Effect.gen(function* () {
1551+
const reads: Array<OrchestratorMcpThreadReadInput> = [];
1552+
const nativeFetch = yield* FetchHttpClient.Fetch;
1553+
const fetch: typeof globalThis.fetch = (input, init) => {
1554+
const body = init?.body instanceof Uint8Array ? new TextDecoder().decode(init.body) : null;
1555+
if (body?.startsWith("{")) {
1556+
const rpc: {
1557+
method?: string;
1558+
params?: { name?: string; arguments?: OrchestratorMcpThreadReadInput };
1559+
} = JSON.parse(body);
1560+
if (
1561+
rpc.method === "tools/call" &&
1562+
rpc.params?.name === "t3_thread_read" &&
1563+
rpc.params.arguments !== undefined
1564+
) {
1565+
reads.push(rpc.params.arguments);
1566+
}
1567+
}
1568+
return nativeFetch(input, init);
1569+
};
1570+
return yield* Effect.scoped(
1571+
Effect.gen(function* () {
1572+
const thread = yield* makeBoxThread;
1573+
const b = yield* serveBox(thread);
1574+
const offers = yield* SubscriptionRef.make<
1575+
ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest>
1576+
>([]);
1577+
const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build);
1578+
const a = yield* makeLaptop(database, offers);
1579+
yield* linkTo(a.links, b, "full-access");
1580+
yield* seedParent(a);
1581+
const result =
1582+
scenario === "long"
1583+
? `Complete ${"detail ".repeat(12_000)}end.`
1584+
: "Authoritative result.";
1585+
const middle = scenario === "later-run" || scenario === "shrinking";
1586+
const history = (count: number, runId: RunId, prefix: string): Array<TimelineEntry> =>
1587+
Array.from({ length: count }, (_, index) => ({
1588+
id: TurnItemId.make(`${prefix}:${index}`),
1589+
runId,
1590+
text: "History",
1591+
type: scenario === "sparse" && index % 2 === 0 ? "error" : "assistant_message",
1592+
}));
1593+
const timeline: Array<TimelineEntry> = [
1594+
...history(middle ? 150 : 300, boxRunId, "before"),
1595+
{
1596+
id: TurnItemId.make("selected"),
1597+
runId: scenario === "no-run" ? boxLaterRunId : boxRunId,
1598+
text: result,
1599+
type: "assistant_message",
1600+
},
1601+
...(middle ? history(200, boxLaterRunId, "later") : []),
1602+
];
1603+
yield* Ref.set(thread.timeline, timeline);
1604+
yield* Ref.set(thread.pageSize, 100);
1605+
if (scenario === "growing") {
1606+
yield* Ref.set(
1607+
thread.beforeTail,
1608+
Ref.set(thread.timeline, [...timeline, ...history(101, boxLaterRunId, "growth")]),
1609+
);
1610+
}
1611+
if (scenario === "shrinking") {
1612+
yield* Ref.set(thread.beforeTail, Ref.set(thread.timeline, timeline.slice(0, 151)));
1613+
}
1614+
if (scenario === "cancel") {
1615+
yield* Ref.set(
1616+
thread.beforeTail,
1617+
Effect.gen(function* () {
1618+
yield* Deferred.succeed(thread.readStarted, undefined);
1619+
yield* Deferred.await(thread.releaseRead);
1620+
}),
1621+
);
1622+
}
1623+
if (scenario === "no-run") yield* Ref.set(thread.runless, true);
1624+
const delegated = yield* delegateToBox(a);
1625+
const status =
1626+
scenario === "cancelled" || scenario === "interrupted" ? scenario : "completed";
1627+
yield* Ref.set(thread.status, status);
1628+
yield* Deferred.succeed(thread.finished, undefined);
1629+
if (scenario === "cancel") {
1630+
yield* Deferred.await(thread.readStarted);
1631+
yield* a.threads.stopDelegatedTasks({
1632+
threadId: parentThreadId,
1633+
commandId: CommandId.make("command:stop:tail-probe"),
1634+
});
1635+
yield* Deferred.succeed(thread.releaseRead, undefined);
1636+
expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ status: "interrupted" });
1637+
yield* Deferred.await(thread.interruptedOnce);
1638+
expect(yield* Ref.get(thread.stops)).toContain("thread.stop");
1639+
} else {
1640+
expect(yield* taskResult(a, delegated.taskId)).toMatchObject({
1641+
status,
1642+
result,
1643+
});
1644+
}
1645+
const pages = reads.filter((read) => read.itemId === undefined);
1646+
const positions =
1647+
scenario === "cancelled" || scenario === "interrupted"
1648+
? [undefined, 99, 199, 299]
1649+
: scenario === "later-run"
1650+
? [undefined, 299, 99, 199, 299]
1651+
: scenario === "shrinking"
1652+
? [undefined, 299, 99]
1653+
: scenario === "growing"
1654+
? [undefined, 299, 99, 199, 299, 399]
1655+
: [undefined, 299];
1656+
expect(pages.map((read) => read.afterPosition)).toEqual(positions);
1657+
expect(
1658+
pages.every(
1659+
(read) =>
1660+
read.view === "messages" && read.limit === 100 && read.maxCharsPerItem === undefined,
1661+
),
1662+
).toBe(true);
1663+
const refills = reads.filter((read) => read.itemId !== undefined);
1664+
expect(refills).toHaveLength(scenario === "long" ? 2 : 0);
1665+
if (scenario === "long")
1666+
expect(refills.map((read) => read.textOffset)).toEqual([20_000, 70_000]);
1667+
}),
1668+
).pipe(
1669+
Effect.provide(NodeHttpServer.layerTest),
1670+
Effect.provideService(FetchHttpClient.Fetch, fetch),
1671+
);
1672+
}),
1673+
);
1674+
15331675
it.effect("keeps a successful remote reply despite an error item in the run's activity", () =>
15341676
Effect.scoped(
15351677
Effect.gen(function* () {

‎apps/server/src/peer/RemoteDelegation.ts‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -441,11 +441,13 @@ const make = Effect.gen(function* () {
441441
remote: OrchestrationV2RemoteTaskChild,
442442
runId: RunId | null,
443443
failed = false,
444+
probeSuccess = true,
444445
) {
445446
let reply: OrchestratorMcpThreadTimelineItem | undefined;
446447
let error: OrchestratorMcpThreadTimelineItem | undefined;
447448
let afterPosition: number | undefined;
448449
let remainingItems = 0;
450+
let tailProbed = false;
449451
while (true) {
450452
const read = yield* forwarding.readThread(scope, remote.environmentId, {
451453
threadId: remote.threadId,
@@ -467,6 +469,25 @@ const make = Effect.gen(function* () {
467469
) ?? error;
468470
}
469471
if (!read.hasMore || read.nextPosition === null) break;
472+
if (!failed && probeSuccess && !tailProbed) {
473+
tailProbed = true;
474+
const tailPosition = Math.floor((read.thread.itemCount - 1) / 100) * 100 - 1;
475+
if (tailPosition > read.nextPosition) {
476+
const tail = yield* forwarding.readThread(scope, remote.environmentId, {
477+
threadId: remote.threadId,
478+
view: "messages",
479+
limit: 100,
480+
afterPosition: tailPosition,
481+
});
482+
const tailReply = tail.items.findLast(
483+
(item) => item.type === "assistant_message" && (runId === null || item.runId === runId),
484+
);
485+
if (!tail.hasMore && tailReply !== undefined) {
486+
reply = tailReply;
487+
break;
488+
}
489+
}
490+
}
470491
afterPosition = read.nextPosition;
471492
remainingItems = read.thread.itemCount - afterPosition - 1;
472493
}
@@ -633,7 +654,13 @@ const make = Effect.gen(function* () {
633654
runId: remote.runId ?? waited.runId,
634655
status: waited.status,
635656
};
636-
const result = yield* lastReply(scope, remote, resultRun.runId, resultRun.status === "failed");
657+
const result = yield* lastReply(
658+
scope,
659+
remote,
660+
resultRun.runId,
661+
resultRun.status === "failed",
662+
resultRun.status === "completed",
663+
);
637664
yield* complete(
638665
parentThreadId,
639666
taskId,

0 commit comments

Comments
 (0)