Skip to content

Commit b92c8f4

Browse files
t3dotggclaude
andauthored
fix(server): token telemetry counts each request once and only confirmed models (pingdotgg#29001)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent dd6c05e commit b92c8f4

32 files changed

Lines changed: 1470 additions & 178 deletions

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5457,6 +5457,7 @@ describe("ClaudeAdapterV2 background wake turns", () => {
54575457
outputTokens: queryExits ? 40 : 46,
54585458
cachedInputTokens: queryExits ? 2000 : 11455,
54595459
cacheCreationTokens: queryExits ? 500 : 2847,
5460+
...(queryExits ? {} : { models: ["claude-sonnet-4-6"] }),
54605461
},
54615462
);
54625463
}

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts‎

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,14 @@ import {
2121
} from "./claudeWorkflowProgress.ts";
2222

2323
import { isWorkspaceImagePreviewPath } from "@t3tools/shared/filePreview";
24-
import { normalizeClaudeTurnTokenUsage } from "../../provider/ClaudeTurnTokenUsage.ts";
24+
import {
25+
addClaudeLedgerUsage,
26+
claudeLedgerDelta,
27+
claudeModelLedger,
28+
normalizeClaudeTurnTokenUsage,
29+
type ClaudeLedgerUsage,
30+
type ClaudeModelLedger,
31+
} from "../../provider/ClaudeTurnTokenUsage.ts";
2532
import { toMcpElicitationResponse } from "../../provider/CodexMcpElicitation.ts";
2633
import {
2734
type CanUseTool,
@@ -4071,6 +4078,9 @@ interface ActiveClaudeTurnContext {
40714078
compactionFailure: { readonly message: string | undefined } | null;
40724079
// Steers end segments whose usage is excluded from the final result; bill them too, oldest first.
40734080
readonly steeredSegmentUsage: Array<SDKResultMessage["usage"]>;
4081+
// Ledger growth of the steered segments; undefined once any segment's growth is unknown.
4082+
steeredLedgerUsage: ClaudeLedgerUsage | undefined;
4083+
readonly rootModels: Set<string>;
40744084
readonly subagentsByTaskId: Map<string, ActiveClaudeSubagent>;
40754085
readonly subagentsByToolUseId: Map<string, ActiveClaudeSubagent>;
40764086
readonly promptUuid: string | null;
@@ -4122,6 +4132,8 @@ interface ClaudeLiveQueryContext {
41224132
permissionMode: PermissionMode;
41234133
stopping: boolean;
41244134
readonly subagentsFromEarlierProcesses: ReadonlySet<ActiveClaudeSubagent>;
4135+
// The SDK's per-process model ledger at the last result; a new process starts at zero.
4136+
modelLedger: ClaudeModelLedger | undefined;
41254137
// Per process, not per session: a relative temp dir follows the cwd this CLI
41264138
// was launched in, and a later turn's cwd can differ.
41274139
readonly taskOutputRoot: string | undefined;
@@ -4775,6 +4787,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
47754787
const interruptedTurns = yield* Ref.make(new Set<OrchestrationV2ProviderTurn["id"]>());
47764788
const steeredTurns = yield* Ref.make(new Set<OrchestrationV2ProviderTurn["id"]>());
47774789
const queryContext = yield* Ref.make<ClaudeLiveQueryContext | null>(null);
4790+
// Wake buffers replay result frames, so ledger growth is measured once on arrival.
4791+
const resultLedgerUsage = new WeakMap<SDKMessage, ClaudeLedgerUsage>();
47784792
// A worktree handoff changes the project settings even within one session.
47794793
let bypassCache: { cwd: string | null; availability: ClaudeBypassAvailability } | null =
47804794
null;
@@ -7613,6 +7627,17 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
76137627
input.context.subagentsByToolUseId.size > 0,
76147628
input.status,
76157629
input.context.steeredSegmentUsage,
7630+
{
7631+
rootModels: input.context.rootModels,
7632+
...(input.result === undefined
7633+
? {}
7634+
: {
7635+
ledger: addClaudeLedgerUsage(
7636+
input.context.steeredLedgerUsage,
7637+
resultLedgerUsage.get(input.result),
7638+
),
7639+
}),
7640+
},
76167641
),
76177642
},
76187643
}),
@@ -8528,6 +8553,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
85288553

85298554
if (message.type === "assistant" && input.replayed !== true) {
85308555
if (message.parent_tool_use_id === null) {
8556+
// CLI notices and API errors arrive as assistant messages from "<synthetic>".
8557+
if (message.message.model && message.message.model !== "<synthetic>") {
8558+
context.rootModels.add(message.message.model);
8559+
}
85318560
// Subagent uuids exist only in the sidechain transcript, so resume and fork cannot target them.
85328561
context.nativeMessageCursor = message.uuid;
85338562
context.latestAssistantRateLimit =
@@ -9409,6 +9438,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
94099438
const wasSteered = (yield* Ref.get(steeredTurns)).has(context.providerTurnId);
94109439
if (!interrupted && wasSteered && isClaudeActiveSteeringAbortResult(message)) {
94119440
context.steeredSegmentUsage.push(message.usage);
9441+
context.steeredLedgerUsage = addClaudeLedgerUsage(
9442+
context.steeredLedgerUsage,
9443+
resultLedgerUsage.get(message),
9444+
);
94129445
return;
94139446
}
94149447
yield* Ref.update(steeredTurns, (current) => {
@@ -10404,6 +10437,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
1040410437
(subagent) => subagent.task.status === "running",
1040510438
),
1040610439
),
10440+
modelLedger: new Map(),
1040710441
taskOutputRoot: claudeTaskOutputRoot(
1040810442
adapterOptions.environment,
1040910443
path,
@@ -10432,6 +10466,16 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
1043210466
) {
1043310467
context.permissionMode = message.permissionMode;
1043410468
}
10469+
if (message.type === "result") {
10470+
const ledger = claudeModelLedger(message.modelUsage);
10471+
const growth = claudeLedgerDelta(context.modelLedger, ledger);
10472+
if (growth !== undefined) resultLedgerUsage.set(message, growth);
10473+
// A ledger that went backwards cannot anchor the next turn's growth.
10474+
context.modelLedger =
10475+
growth === undefined && context.modelLedger !== undefined
10476+
? undefined
10477+
: ledger;
10478+
}
1043510479
return handleSdkMessage({ query: querySession, message });
1043610480
}),
1043710481
),
@@ -10577,6 +10621,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
1057710621
latestAssistantRateLimit: null,
1057810622
compactionFailure: null,
1057910623
steeredSegmentUsage: [],
10624+
steeredLedgerUsage: { inputTokens: 0, outputTokens: 0 },
10625+
rootModels: new Set(),
1058010626
subagentsByTaskId: new Map(),
1058110627
subagentsByToolUseId: new Map(),
1058210628
promptUuid,

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1830,6 +1830,17 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
18301830
const turnWaiters = yield* Ref.make(new Map<string, Deferred.Deferred<void, never>>());
18311831
const subagentThreads = yield* Ref.make(new Map<string, CodexSubagentThreadContext>());
18321832
const subagentSelections = new Map<string, Omit<ModelSelection, "instanceId">>();
1833+
// A reroute names one turn; until new settings arrive, the thread's model is unknown.
1834+
const reroutedTurnModels = new Map<string, string>();
1835+
const reroutedThreads = new Set<string>();
1836+
const reportedTurnModel = (nativeThreadId: string, nativeTurnId: string) => {
1837+
const rerouted = reroutedTurnModels.get(nativeTurnId);
1838+
reroutedTurnModels.delete(nativeTurnId);
1839+
if (rerouted !== undefined) return rerouted;
1840+
return reroutedThreads.has(nativeThreadId)
1841+
? undefined
1842+
: subagentSelections.get(nativeThreadId)?.model;
1843+
};
18331844
const pendingSubagentTurns = yield* Ref.make(
18341845
new Map<string, ReadonlyArray<PendingCodexSubagentTurnStarted>>(),
18351846
);
@@ -4475,15 +4486,22 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
44754486
);
44764487

44774488
yield* client.handleServerNotification("thread/settings/updated", (payload) =>
4478-
updateSubagentSelection(
4479-
payload.threadId,
4480-
payload.threadSettings.model,
4481-
payload.threadSettings.effort,
4482-
payload.threadSettings.serviceTier,
4489+
Effect.sync(() => reroutedThreads.delete(payload.threadId)).pipe(
4490+
Effect.andThen(
4491+
updateSubagentSelection(
4492+
payload.threadId,
4493+
payload.threadSettings.model,
4494+
payload.threadSettings.effort,
4495+
payload.threadSettings.serviceTier,
4496+
),
4497+
),
44834498
),
44844499
);
44854500
yield* client.handleServerNotification("model/rerouted", (payload) =>
4486-
updateSubagentSelection(payload.threadId, payload.toModel),
4501+
Effect.sync(() => {
4502+
reroutedTurnModels.set(payload.turnId, payload.toModel);
4503+
reroutedThreads.add(payload.threadId);
4504+
}).pipe(Effect.andThen(updateSubagentSelection(payload.threadId, payload.toModel))),
44874505
);
44884506

44894507
yield* client.handleServerNotification("hook/completed", (payload) =>
@@ -6127,6 +6145,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
61276145
),
61286146
input.nativeTurnId,
61296147
input.status === "completed",
6148+
reportedTurnModel(
6149+
input.context.providerThread.nativeThreadRef?.nativeId ?? "",
6150+
input.nativeTurnId,
6151+
),
61306152
),
61316153
};
61326154
if (!holdsForGoal) {

‎apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2150,6 +2150,20 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
21502150
const { runtime, thread } = yield* resumed([
21512151
out("session.prompt", { sessionID: SESSION, id: PROMPT_ID, text: "<any>" }),
21522152
promptAccepted,
2153+
event("session.step.started", {
2154+
sessionID: SESSION,
2155+
assistantMessageID: "msg_before_gap",
2156+
agent: "build",
2157+
model: { id: "big-pickle", providerID: "opencode", variant: "default" },
2158+
started: 1,
2159+
}),
2160+
event("session.step.ended", {
2161+
sessionID: SESSION,
2162+
assistantMessageID: "msg_before_gap",
2163+
finish: "tool-calls",
2164+
cost: 0,
2165+
tokens: { input: 100, output: 10, reasoning: 0, cache: { read: 0, write: 0 } },
2166+
}),
21532167
...reconnected({}, "succeeded"),
21542168
]);
21552169
const events = yield* runtime.events.pipe(
@@ -2164,6 +2178,15 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
21642178
"command_execution:completed",
21652179
]);
21662180
assert.deepInclude(collected.at(-1), { type: "turn.terminal", status: "completed" });
2181+
// Steps that ended during the gap sent no tokens or models, so both stay incomplete.
2182+
const settled = collected.findLast(
2183+
(event) =>
2184+
event.type === "provider_turn.updated" && event.providerTurn.completedAt !== null,
2185+
);
2186+
const usage =
2187+
settled?.type === "provider_turn.updated" ? settled.providerTurn.turnTokenUsage : undefined;
2188+
assert.deepInclude(usage, { usageStatus: "partial", inputTokens: 100, outputTokens: 10 });
2189+
assert.notProperty(usage, "models");
21672190
}).pipe(Effect.scoped),
21682191
);
21692192

@@ -2879,6 +2902,40 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
28792902
}).pipe(Effect.scoped),
28802903
);
28812904

2905+
it.effect("leaves models unknown when a step ends without a seen start", () =>
2906+
Effect.gen(function* () {
2907+
const tokens = { input: 10, output: 1, reasoning: 0, cache: { read: 0, write: 0 } };
2908+
const step = (id: string) => ({ sessionID: SESSION, assistantMessageID: id });
2909+
const { runtime, thread } = yield* resumed([
2910+
out("session.prompt", { sessionID: SESSION, text: "<any>" }),
2911+
promptAccepted,
2912+
event("session.step.started", {
2913+
...step("msg_seen"),
2914+
agent: "build",
2915+
model: { id: "big-pickle", providerID: "opencode", variant: "default" },
2916+
started: 1,
2917+
}),
2918+
event("session.step.ended", { ...step("msg_seen"), finish: "tool-calls", cost: 0, tokens }),
2919+
event("session.step.ended", { ...step("msg_unseen"), finish: "stop", cost: 0, tokens }),
2920+
event("session.execution.succeeded", { sessionID: SESSION }),
2921+
]);
2922+
const turn = yield* runtime.events.pipe(
2923+
Stream.filter(
2924+
(event) =>
2925+
event.type === "provider_turn.updated" && event.providerTurn.completedAt !== null,
2926+
),
2927+
Stream.runHead,
2928+
Effect.forkScoped,
2929+
);
2930+
yield* runtime.startTurn(turnInput(thread));
2931+
const settled = Option.getOrUndefined(yield* Fiber.join(turn));
2932+
const usage =
2933+
settled?.type === "provider_turn.updated" ? settled.providerTurn.turnTokenUsage : undefined;
2934+
assert.deepInclude(usage, { usageStatus: "complete", inputTokens: 20, outputTokens: 2 });
2935+
assert.notProperty(usage, "models");
2936+
}).pipe(Effect.scoped),
2937+
);
2938+
28822939
it.effect("reports cache writes as cache creation, not only as input", () =>
28832940
Effect.gen(function* () {
28842941
const { runtime, thread } = yield* resumed([

‎apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts‎

Lines changed: 10 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ import * as EventStore from "./EventStore.ts";
4545
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
4646
import * as ProjectionStore from "./ProjectionStore.ts";
4747
import * as ProviderEventIngestor from "./ProviderEventIngestor.ts";
48+
import * as ProviderUsageAnalytics from "../telemetry/ProviderUsageAnalytics.ts";
4849
import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
4950
import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts";
5051
import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
@@ -325,12 +326,14 @@ it.effect("refreshes the failing instance snapshot after a committed usage_limit
325326
);
326327

327328
it.effect("records accepted billed turn usage once without billing the context window", () => {
328-
const recorded: Array<Readonly<Record<string, unknown>>> = [];
329-
const layerAnalytics = Layer.succeed(ProviderEventIngestor.ProviderTurnAnalytics, {
330-
record: (properties: Readonly<Record<string, unknown>>) =>
329+
const recorded: Array<ProviderUsageAnalytics.ProviderTurnReport> = [];
330+
const layerAnalytics = Layer.succeed(ProviderUsageAnalytics.ProviderTurnAnalytics, {
331+
runStarted: () => Effect.void,
332+
turnCompleted: (report) =>
331333
Effect.sync(() => {
332-
recorded.push(properties);
334+
recorded.push(report);
333335
}),
336+
runSettled: () => Effect.void,
334337
});
335338
return Effect.gen(function* () {
336339
const now = yield* DateTime.now;
@@ -413,23 +416,9 @@ it.effect("records accepted billed turn usage once without billing the context w
413416
assert.isEmpty(ignored);
414417
assert.lengthOf(recorded, 1);
415418
assert.deepEqual(recorded[0], {
416-
provider: CODEX_DRIVER,
417-
terminalStatus: "completed",
418-
usageStatus: "complete",
419-
usageScope: "main_agent",
420-
hasSubagents: false,
421-
inputTokens: 40,
422-
cachedInputTokens: 30,
423-
outputTokens: 10,
424-
model: modelSelection.model,
425-
mixedModels: false,
426-
runtimeMode: "full-access",
427-
interactionMode: "default",
428-
createdBy: "agent",
429-
creationSource: "provider",
430-
scheduled: false,
431-
subagent: true,
432-
durationMs: 120,
419+
driver: CODEX_DRIVER,
420+
providerTurn,
421+
context: input.analyticsContext,
433422
});
434423
}).pipe(Effect.provide(layerTest.pipe(Layer.provide(layerAnalytics))));
435424
});

0 commit comments

Comments
 (0)