Skip to content

Commit e1726c6

Browse files
committed
perf(grok): cancel pending completions without quadratic copies
1 parent 7195b69 commit e1726c6

2 files changed

Lines changed: 58 additions & 59 deletions

File tree

‎packages/provider-grok/src/server/xaiAcpExtension.test.ts‎

Lines changed: 51 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1353,44 +1353,57 @@ describe("XAiAcpExtension", () => {
13531353
}),
13541354
);
13551355

1356-
it.effect("cancels pending completions without restarting a stalled runtime", () =>
1357-
Effect.gen(function* () {
1358-
const hungStart = yield* Deferred.make<never>();
1359-
const hungPrompt = yield* Deferred.make<never>();
1360-
let startCount = 0;
1361-
let cancelCalled = false;
1362-
const baseRuntime = {
1363-
start: () => {
1364-
startCount += 1;
1365-
return startCount === 1
1366-
? Effect.succeed({
1367-
sessionId: "root-session",
1368-
initializeResult: {},
1369-
sessionSetupResult: {},
1370-
modelConfigId: undefined,
1371-
})
1372-
: Deferred.await(hungStart);
1373-
},
1374-
prompt: () => Deferred.await(hungPrompt),
1375-
cancel: Effect.sync(() => {
1376-
cancelCalled = true;
1377-
}),
1378-
handleExtNotification: () => Effect.void,
1379-
handleExtRequest: () => Effect.void,
1380-
} as unknown as AcpSessionRuntime.AcpSessionRuntime["Service"];
1381-
1382-
const runtime = yield* makeXAiPromptCompletionRuntime(baseRuntime);
1383-
const promptFiber = yield* runtime
1384-
.prompt({ prompt: [{ type: "text", text: "hi" }] })
1385-
.pipe(Effect.forkChild);
1386-
yield* Effect.yieldNow;
1387-
1388-
yield* runtime.cancel;
1389-
1390-
expect(cancelCalled).toBe(true);
1391-
expect(startCount).toBe(1);
1392-
expect((yield* Fiber.join(promptFiber)).stopReason).toBe("cancelled");
1393-
}),
1356+
it.effect.each([0, 1, 128])(
1357+
"cancels %i pending completions without restarting a stalled runtime",
1358+
(promptCount) =>
1359+
Effect.gen(function* () {
1360+
const hungStart = yield* Deferred.make<never>();
1361+
const hungPrompt = yield* Deferred.make<never>();
1362+
const promptsReady = yield* Deferred.make<void>();
1363+
let enteredPrompts = 0;
1364+
let startCount = 0;
1365+
let cancelCount = 0;
1366+
const baseRuntime = {
1367+
start: () => {
1368+
startCount += 1;
1369+
return startCount <= promptCount
1370+
? Effect.succeed({
1371+
sessionId: "root-session",
1372+
initializeResult: {},
1373+
sessionSetupResult: {},
1374+
modelConfigId: undefined,
1375+
})
1376+
: Deferred.await(hungStart);
1377+
},
1378+
prompt: () =>
1379+
Effect.gen(function* () {
1380+
enteredPrompts += 1;
1381+
if (enteredPrompts === promptCount) yield* Deferred.succeed(promptsReady, undefined);
1382+
return yield* Deferred.await(hungPrompt);
1383+
}),
1384+
cancel: Effect.sync(() => {
1385+
cancelCount += 1;
1386+
}),
1387+
handleExtNotification: () => Effect.void,
1388+
handleExtRequest: () => Effect.void,
1389+
} as unknown as AcpSessionRuntime.AcpSessionRuntime["Service"];
1390+
1391+
const runtime = yield* makeXAiPromptCompletionRuntime(baseRuntime);
1392+
const promptFibers = yield* Effect.forEach(Array.from({ length: promptCount }), () =>
1393+
runtime.prompt({ prompt: [{ type: "text", text: "hi" }] }).pipe(Effect.forkChild),
1394+
);
1395+
if (promptCount > 0) yield* Deferred.await(promptsReady);
1396+
1397+
yield* runtime.cancel;
1398+
yield* runtime.cancel;
1399+
1400+
expect(cancelCount).toBe(2);
1401+
expect(startCount).toBe(promptCount);
1402+
const responses = yield* Effect.forEach(promptFibers, Fiber.join);
1403+
expect(responses.map((response) => response.stopReason)).toEqual(
1404+
Array.from({ length: promptCount }, () => "cancelled"),
1405+
);
1406+
}),
13941407
);
13951408
});
13961409

‎packages/provider-grok/src/server/xaiAcpExtension.ts‎

Lines changed: 7 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1313,24 +1313,11 @@ const unregisterXAiPromptCompletionFallback = (
13131313

13141314
const abortPendingPromptCompletions = (
13151315
pendingRef: Ref.Ref<ReadonlyArray<PendingXAiPromptCompletion>>,
1316-
sessionId?: string,
13171316
) =>
1318-
Ref.modify(pendingRef, (pending) => {
1319-
const [toAbort, remaining] = pending.reduce<
1320-
[ReadonlyArray<PendingXAiPromptCompletion>, ReadonlyArray<PendingXAiPromptCompletion>]
1321-
>(
1322-
([aborting, kept], entry) =>
1323-
sessionId === undefined || entry.sessionId === sessionId
1324-
? [[...aborting, entry], kept]
1325-
: [aborting, [...kept, entry]],
1326-
[[], []],
1327-
);
1328-
if (toAbort.length === 0) {
1329-
return [Effect.void, pending] as const;
1330-
}
1331-
return [
1317+
Ref.getAndSet(pendingRef, []).pipe(
1318+
Effect.flatMap((pending) =>
13321319
Effect.forEach(
1333-
toAbort,
1320+
pending,
13341321
(entry) =>
13351322
Deferred.succeed(
13361323
entry.deferred,
@@ -1341,11 +1328,10 @@ const abortPendingPromptCompletions = (
13411328
agentResult: null,
13421329
}),
13431330
),
1344-
{ concurrency: "unbounded" },
1345-
).pipe(Effect.asVoid),
1346-
remaining,
1347-
] as const;
1348-
}).pipe(Effect.flatten);
1331+
{ concurrency: "unbounded", discard: true },
1332+
),
1333+
),
1334+
);
13491335

13501336
const resolveXAiPromptCompletionFallback = ({
13511337
pendingRef,

0 commit comments

Comments
 (0)