Skip to content

Commit 6ab3b95

Browse files
committed
fix(server): record ACP prompt response turn usage
1 parent 0080e80 commit 6ab3b95

2 files changed

Lines changed: 369 additions & 2 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts‎

Lines changed: 287 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ import {
7474
import type { ProviderContinuationRequest } from "../ProviderContinuationRequests.ts";
7575
import {
7676
AcpProviderCapabilitiesV2,
77+
acpPromptResponseTurnTokenUsage,
7778
acpProviderItemNativeId,
7879
acpScopedNativeId,
7980
acpCarryoverTerminalShouldClearContinuation,
@@ -108,6 +109,95 @@ const testLayer = Layer.mergeAll(NodeServices.layer, IdAllocator.layer, serverCo
108109
const ACP_TEST_DRIVER = ProviderDriverKind.make("acp-test");
109110
const decodeUnknownJson = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Unknown));
110111

112+
describe("acpPromptResponseTurnTokenUsage", () => {
113+
it("projects ACP prompt usage into complete main-agent turn usage", () => {
114+
assert.deepEqual(
115+
acpPromptResponseTurnTokenUsage(
116+
{
117+
inputTokens: 60,
118+
outputTokens: 15,
119+
totalTokens: 75,
120+
cachedReadTokens: 40,
121+
cachedWriteTokens: 1,
122+
thoughtTokens: 4,
123+
},
124+
{
125+
inputTokens: 20,
126+
outputTokens: 5,
127+
totalTokens: 25,
128+
cachedReadTokens: 10,
129+
cachedWriteTokens: 0,
130+
thoughtTokens: 1,
131+
},
132+
true,
133+
"completed",
134+
),
135+
{
136+
usageScope: "main_agent",
137+
usageStatus: "complete",
138+
hasSubagents: true,
139+
inputTokens: 40,
140+
outputTokens: 10,
141+
cachedInputTokens: 30,
142+
cacheCreationTokens: 1,
143+
reasoningTokens: 3,
144+
},
145+
);
146+
});
147+
148+
it("returns unavailable usage when ACP omits prompt usage", () => {
149+
assert.deepEqual(acpPromptResponseTurnTokenUsage(undefined, null, false, "failed"), {
150+
usageScope: "main_agent",
151+
usageStatus: "unavailable",
152+
hasSubagents: false,
153+
});
154+
});
155+
156+
it("marks observed usage partial when the turn does not complete", () => {
157+
assert.deepEqual(
158+
acpPromptResponseTurnTokenUsage(
159+
{ inputTokens: 60, outputTokens: 15, totalTokens: 75 },
160+
{ inputTokens: 40, outputTokens: 10, totalTokens: 50 },
161+
false,
162+
"interrupted",
163+
),
164+
{
165+
usageScope: "main_agent",
166+
usageStatus: "partial",
167+
hasSubagents: false,
168+
inputTokens: 20,
169+
outputTokens: 5,
170+
},
171+
);
172+
});
173+
174+
it("does not attribute resumed session totals or counter resets to one turn", () => {
175+
const current = { inputTokens: 60, outputTokens: 15, totalTokens: 75 };
176+
const unavailable = {
177+
usageScope: "main_agent",
178+
usageStatus: "unavailable",
179+
hasSubagents: false,
180+
};
181+
assert.deepEqual(
182+
acpPromptResponseTurnTokenUsage(current, null, false, "completed"),
183+
unavailable,
184+
);
185+
assert.deepEqual(
186+
acpPromptResponseTurnTokenUsage(
187+
current,
188+
{
189+
inputTokens: 90,
190+
outputTokens: 15,
191+
totalTokens: 105,
192+
},
193+
false,
194+
"completed",
195+
),
196+
unavailable,
197+
);
198+
});
199+
});
200+
111201
describe("acpProjectedCommandExitCode", () => {
112202
const successOutput = { type: "Bash", exit_code: 0 };
113203
const failedOutput = { type: "Bash", exit_code: 1 };
@@ -604,6 +694,203 @@ function makeTurnInput(input: {
604694
}
605695

606696
describe("AcpAdapterV2", () => {
697+
it.live(
698+
"projects session usage deltas without attributing missing responses to later turns",
699+
() =>
700+
Effect.gen(function* () {
701+
const path = yield* Path.Path;
702+
const instanceId = ProviderInstanceId.make("acp-test-prompt-usage");
703+
const threadId = ThreadId.make("thread-acp-prompt-usage");
704+
let promptOrdinal = 0;
705+
const promptUsages = [
706+
{ inputTokens: 60, outputTokens: 15, totalTokens: 75 },
707+
{ inputTokens: 90, outputTokens: 22, totalTokens: 112 },
708+
undefined,
709+
{ inputTokens: 130, outputTokens: 30, totalTokens: 160 },
710+
{ inputTokens: 150, outputTokens: 40, totalTokens: 190 },
711+
];
712+
const adapter = makeAcpAdapterV2({
713+
crypto: yield* Crypto.Crypto,
714+
instanceId,
715+
fileSystem: yield* FileSystem.FileSystem,
716+
idAllocator: yield* IdAllocator.IdAllocatorV2,
717+
serverConfig: yield* ServerConfig.ServerConfig,
718+
selfInvocation: yield* resolveSelfInvocation(),
719+
flavor: {
720+
driver: ACP_TEST_DRIVER,
721+
capabilities: AcpProviderCapabilitiesV2,
722+
makeRuntime: makeMockRuntime({
723+
childProcessSpawner: yield* ChildProcessSpawner.ChildProcessSpawner,
724+
mockAgentPath: yield* path.fromFileUrl(
725+
new URL("../../../scripts/acp-mock-agent.ts", import.meta.url),
726+
),
727+
wrapRuntime: (runtime) => ({
728+
...runtime,
729+
prompt: () => {
730+
promptOrdinal += 1;
731+
return Effect.succeed({
732+
stopReason: "end_turn" as const,
733+
usage: promptUsages[promptOrdinal - 1] ?? null,
734+
});
735+
},
736+
}),
737+
}),
738+
},
739+
});
740+
const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({
741+
runtimeMode: "full-access",
742+
interactionMode: "default",
743+
cwd: process.cwd(),
744+
});
745+
const modelSelection = { instanceId, model: "default" } as const;
746+
const runtime = yield* adapter.openSession({
747+
threadId,
748+
providerSessionId: ProviderSessionId.make("provider-session-acp-prompt-usage"),
749+
modelSelection,
750+
runtimePolicy,
751+
});
752+
const providerThread = yield* runtime.ensureThread({
753+
threadId,
754+
modelSelection,
755+
runtimePolicy,
756+
});
757+
const now = yield* DateTime.now;
758+
const expected = [
759+
{ inputTokens: 60, outputTokens: 15 },
760+
{ inputTokens: 30, outputTokens: 7 },
761+
null,
762+
null,
763+
{ inputTokens: 20, outputTokens: 10 },
764+
];
765+
for (const ordinal of [1, 2, 3, 4, 5]) {
766+
const expectedUsage = expected[ordinal - 1];
767+
if (expectedUsage === undefined) return yield* Effect.die("Missing usage expectation");
768+
yield* runtime.startTurn(
769+
makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal }),
770+
);
771+
const events = Array.from(
772+
yield* runtime.events.pipe(
773+
Stream.takeUntil((event) => event.type === "turn.terminal"),
774+
Stream.runCollect,
775+
),
776+
);
777+
const completed = events.find(
778+
(event) =>
779+
event.type === "provider_turn.updated" && event.providerTurn.status === "completed",
780+
);
781+
assert.deepEqual(
782+
completed?.type === "provider_turn.updated"
783+
? completed.providerTurn.turnTokenUsage
784+
: null,
785+
expectedUsage === null
786+
? { usageScope: "main_agent", usageStatus: "unavailable", hasSubagents: false }
787+
: {
788+
usageScope: "main_agent",
789+
usageStatus: "complete",
790+
hasSubagents: false,
791+
...expectedUsage,
792+
},
793+
);
794+
}
795+
}).pipe(Effect.provide(testLayer), Effect.scoped),
796+
);
797+
798+
it.live("retains returned usage when Stop finalizes before the prompt callback", () =>
799+
Effect.gen(function* () {
800+
const path = yield* Path.Path;
801+
const instanceId = ProviderInstanceId.make("acp-test-prompt-usage-stop");
802+
const threadId = ThreadId.make("thread-acp-prompt-usage-stop");
803+
const wireSettled = yield* Deferred.make<void>();
804+
const releaseCompletion = yield* Deferred.make<void>();
805+
const idAllocator = yield* IdAllocator.IdAllocatorV2;
806+
const adapter = makeAcpAdapterV2({
807+
crypto: yield* Crypto.Crypto,
808+
instanceId,
809+
fileSystem: yield* FileSystem.FileSystem,
810+
idAllocator,
811+
serverConfig: yield* ServerConfig.ServerConfig,
812+
selfInvocation: yield* resolveSelfInvocation(),
813+
testHooks: {
814+
afterPromptWireSettled: () =>
815+
Deferred.succeed(wireSettled, undefined).pipe(
816+
Effect.andThen(Deferred.await(releaseCompletion)),
817+
),
818+
},
819+
flavor: {
820+
driver: ACP_TEST_DRIVER,
821+
capabilities: AcpProviderCapabilitiesV2,
822+
preserveRuntimeOnSettledInterrupt: true,
823+
makeRuntime: makeMockRuntime({
824+
childProcessSpawner: yield* ChildProcessSpawner.ChildProcessSpawner,
825+
mockAgentPath: yield* path.fromFileUrl(
826+
new URL("../../../scripts/acp-mock-agent.ts", import.meta.url),
827+
),
828+
wrapRuntime: (runtime) => ({
829+
...runtime,
830+
prompt: () =>
831+
Effect.succeed({
832+
stopReason: "end_turn" as const,
833+
usage: { inputTokens: 60, outputTokens: 15, totalTokens: 75 },
834+
}),
835+
}),
836+
}),
837+
},
838+
});
839+
const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({
840+
runtimeMode: "full-access",
841+
interactionMode: "default",
842+
cwd: process.cwd(),
843+
});
844+
const modelSelection = { instanceId, model: "default" } as const;
845+
const runtime = yield* adapter.openSession({
846+
threadId,
847+
providerSessionId: ProviderSessionId.make("provider-session-acp-prompt-usage-stop"),
848+
modelSelection,
849+
runtimePolicy,
850+
});
851+
const providerThread = yield* runtime.ensureThread({
852+
threadId,
853+
modelSelection,
854+
runtimePolicy,
855+
});
856+
yield* runtime.startTurn(
857+
makeTurnInput({
858+
threadId,
859+
providerThread,
860+
instanceId,
861+
runtimePolicy,
862+
now: yield* DateTime.now,
863+
}),
864+
);
865+
yield* Deferred.await(wireSettled);
866+
const providerTurnId = idAllocator.derive.providerTurn({
867+
driver: ACP_TEST_DRIVER,
868+
nativeTurnId: acpScopedNativeId(instanceId, "mock-session-1:turn:1"),
869+
});
870+
yield* runtime.interruptTurn({ providerThread, providerTurnId });
871+
const completed = Array.from(
872+
yield* runtime.events.pipe(
873+
Stream.takeUntil((event) => event.type === "turn.terminal"),
874+
Stream.runCollect,
875+
),
876+
).find(
877+
(event) =>
878+
event.type === "provider_turn.updated" && event.providerTurn.status === "interrupted",
879+
);
880+
assert.deepEqual(
881+
completed?.type === "provider_turn.updated" ? completed.providerTurn.turnTokenUsage : null,
882+
{
883+
usageScope: "main_agent",
884+
usageStatus: "partial",
885+
hasSubagents: false,
886+
inputTokens: 60,
887+
outputTokens: 15,
888+
},
889+
);
890+
yield* Deferred.succeed(releaseCompletion, undefined);
891+
}).pipe(Effect.provide(testLayer), Effect.scoped),
892+
);
893+
607894
it.live.each(["failed", "recovered", "completed", "cancelled"] as const)(
608895
"projects Mistral retry notices and their %s outcome",
609896
(outcome) =>

0 commit comments

Comments
 (0)