Skip to content

Commit 53580e1

Browse files
snowzlmbotsteipete
snowzlmbot
and
steipete
authored
fix(usage): preserve provider-billed zero totals (#101177)
* fix(usage): preserve provider-billed zero totals * fix(usage): harden provider-billed cost provenance * fix(openrouter): retry delayed generation metadata * fix(openrouter): satisfy retry lint * refactor(openrouter): consume streamed billed cost * chore: keep release notes out of contributor PR --------- Co-authored-by: snowzlmbot <[email protected]> Co-authored-by: Peter Steinberger <[email protected]>
1 parent 70c2e83 commit 53580e1

15 files changed

Lines changed: 292 additions & 422 deletions

‎extensions/openrouter/index.test.ts‎

Lines changed: 0 additions & 224 deletions
Original file line numberDiff line numberDiff line change
@@ -36,35 +36,6 @@ import {
3636
} from "./provider-catalog.js";
3737
import { resolveThinkingProfile } from "./provider-policy-api.js";
3838

39-
function createOpenRouterDoneStream(params: { responseId: string; totalCost: number }) {
40-
const stream = createAssistantMessageEventStream();
41-
queueMicrotask(() => {
42-
stream.push({
43-
type: "done",
44-
reason: "stop",
45-
message: {
46-
role: "assistant",
47-
api: "openai-completions",
48-
provider: "openrouter",
49-
model: "openrouter/auto",
50-
content: [{ type: "text", text: "ok" }],
51-
responseId: params.responseId,
52-
stopReason: "stop",
53-
timestamp: Date.now(),
54-
usage: {
55-
input: 1,
56-
output: 1,
57-
cacheRead: 0,
58-
cacheWrite: 0,
59-
totalTokens: 2,
60-
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: params.totalCost },
61-
},
62-
} as never,
63-
});
64-
});
65-
return stream;
66-
}
67-
6839
function createOpenRouterDoneStreamWithoutGeneration() {
6940
const stream = createAssistantMessageEventStream();
7041
queueMicrotask(() => {
@@ -85,35 +56,6 @@ function createOpenRouterDoneStreamWithoutGeneration() {
8556
return stream;
8657
}
8758

88-
function createOpenRouterAbortedStream() {
89-
const stream = createAssistantMessageEventStream();
90-
queueMicrotask(() => {
91-
stream.push({
92-
type: "error",
93-
reason: "aborted",
94-
error: {
95-
role: "assistant",
96-
api: "openai-completions",
97-
provider: "openrouter",
98-
model: "openrouter/auto",
99-
content: [],
100-
responseId: "gen-aborted",
101-
stopReason: "aborted",
102-
timestamp: Date.now(),
103-
usage: {
104-
input: 1,
105-
output: 1,
106-
cacheRead: 0,
107-
cacheWrite: 0,
108-
totalTokens: 2,
109-
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0.001 },
110-
},
111-
} as never,
112-
});
113-
});
114-
return stream;
115-
}
116-
11759
type OpenRouterManifest = {
11860
providerAuthChoices?: Array<{
11961
provider?: string;
@@ -808,172 +750,6 @@ describe("openrouter provider hooks", () => {
808750
expect(headers.get("x-openrouter-title")).toBe("OpenClaw");
809751
});
810752

811-
it("reconciles OpenRouter streamed usage with generation metadata cost", async () => {
812-
const provider = await registerSingleProviderPlugin(openrouterPlugin);
813-
const fetchMock = vi.fn(async (url: string) => {
814-
expect(url).toBe("https://openrouter.ai/api/v1/generation?id=gen-cost-1");
815-
return new Response(JSON.stringify({ data: { total_cost: 0.0042 } }), {
816-
headers: { "Content-Type": "application/json" },
817-
status: 200,
818-
});
819-
});
820-
vi.stubGlobal("fetch", fetchMock);
821-
const baseStreamFn = vi.fn(() =>
822-
createOpenRouterDoneStream({ responseId: "gen-cost-1", totalCost: 0.001 }),
823-
);
824-
825-
try {
826-
const wrapped = provider.wrapStreamFn?.({
827-
provider: "openrouter",
828-
modelId: "openrouter/auto",
829-
streamFn: baseStreamFn,
830-
} as never);
831-
if (!wrapped) {
832-
throw new Error("expected OpenRouter wrapper");
833-
}
834-
const stream = await wrapped(
835-
{
836-
provider: "openrouter",
837-
api: "openai-completions",
838-
id: "openrouter/auto",
839-
baseUrl: "https://openrouter.ai/api/v1",
840-
compat: {},
841-
} as never,
842-
{ messages: [] } as never,
843-
{ apiKey: "or-test-key" } as never,
844-
);
845-
const message = await stream.result();
846-
847-
expect(fetchMock).toHaveBeenCalledOnce();
848-
expect(message.usage.cost.total).toBe(0.0042);
849-
} finally {
850-
vi.unstubAllGlobals();
851-
}
852-
});
853-
854-
it("falls back to streamed cost estimate when generation metadata response is oversized", async () => {
855-
const provider = await registerSingleProviderPlugin(openrouterPlugin);
856-
// Body exceeds the 16 MiB cap; readProviderJsonResponse must reject it and
857-
// applyOpenRouterBilledCost must fall back to the streamed estimate.
858-
const oversizedBody = new ReadableStream<Uint8Array>({
859-
start(controller) {
860-
controller.enqueue(new Uint8Array(16 * 1024 * 1024 + 1).fill(0x78));
861-
controller.close();
862-
},
863-
});
864-
const fetchMock = vi.fn(async (url: string) => {
865-
expect(url).toBe("https://openrouter.ai/api/v1/generation?id=gen-oversized-1");
866-
return new Response(oversizedBody, {
867-
headers: { "Content-Type": "application/json" },
868-
status: 200,
869-
});
870-
});
871-
vi.stubGlobal("fetch", fetchMock);
872-
const baseStreamFn = vi.fn(() =>
873-
createOpenRouterDoneStream({ responseId: "gen-oversized-1", totalCost: 0.001 }),
874-
);
875-
876-
try {
877-
const wrapped = provider.wrapStreamFn?.({
878-
provider: "openrouter",
879-
modelId: "openrouter/auto",
880-
streamFn: baseStreamFn,
881-
} as never);
882-
if (!wrapped) {
883-
throw new Error("expected OpenRouter wrapper");
884-
}
885-
const stream = await wrapped(
886-
{
887-
provider: "openrouter",
888-
api: "openai-completions",
889-
id: "openrouter/auto",
890-
baseUrl: "https://openrouter.ai/api/v1",
891-
compat: {},
892-
} as never,
893-
{ messages: [] } as never,
894-
{ apiKey: "or-test-key" } as never,
895-
);
896-
const message = await stream.result();
897-
898-
expect(fetchMock).toHaveBeenCalledOnce();
899-
expect(message.usage.cost.total).toBe(0.001);
900-
} finally {
901-
vi.unstubAllGlobals();
902-
}
903-
});
904-
905-
it("does not fetch generation metadata for custom OpenRouter-compatible routes", async () => {
906-
const provider = await registerSingleProviderPlugin(openrouterPlugin);
907-
const fetchMock = vi.fn();
908-
vi.stubGlobal("fetch", fetchMock);
909-
const baseStreamFn = vi.fn(() =>
910-
createOpenRouterDoneStream({ responseId: "gen-custom-route", totalCost: 0.001 }),
911-
);
912-
913-
try {
914-
const wrapped = provider.wrapStreamFn?.({
915-
provider: "openrouter",
916-
modelId: "openrouter/auto",
917-
streamFn: baseStreamFn,
918-
} as never);
919-
if (!wrapped) {
920-
throw new Error("expected OpenRouter wrapper");
921-
}
922-
const stream = await wrapped(
923-
{
924-
provider: "openrouter",
925-
api: "openai-completions",
926-
id: "openrouter/auto",
927-
baseUrl: "https://proxy.example.test/api/v1",
928-
compat: {},
929-
} as never,
930-
{ messages: [] } as never,
931-
{ apiKey: "or-test-key" } as never,
932-
);
933-
const message = await stream.result();
934-
935-
expect(fetchMock).not.toHaveBeenCalled();
936-
expect(message.usage.cost.total).toBe(0.001);
937-
} finally {
938-
vi.unstubAllGlobals();
939-
}
940-
});
941-
942-
it("does not fetch generation metadata for aborted stream errors", async () => {
943-
const provider = await registerSingleProviderPlugin(openrouterPlugin);
944-
const fetchMock = vi.fn();
945-
vi.stubGlobal("fetch", fetchMock);
946-
const baseStreamFn = vi.fn(() => createOpenRouterAbortedStream());
947-
948-
try {
949-
const wrapped = provider.wrapStreamFn?.({
950-
provider: "openrouter",
951-
modelId: "openrouter/auto",
952-
streamFn: baseStreamFn,
953-
} as never);
954-
if (!wrapped) {
955-
throw new Error("expected OpenRouter wrapper");
956-
}
957-
const stream = await wrapped(
958-
{
959-
provider: "openrouter",
960-
api: "openai-completions",
961-
id: "openrouter/auto",
962-
baseUrl: "https://openrouter.ai/api/v1",
963-
compat: {},
964-
} as never,
965-
{ messages: [] } as never,
966-
{ apiKey: "or-test-key" } as never,
967-
);
968-
const message = await stream.result();
969-
970-
expect(fetchMock).not.toHaveBeenCalled();
971-
expect(message.stopReason).toBe("aborted");
972-
} finally {
973-
vi.unstubAllGlobals();
974-
}
975-
});
976-
977753
it("merges resolved OpenRouter model params into transport params", async () => {
978754
const provider = await registerSingleProviderPlugin(openrouterPlugin);
979755
const patch = provider.extraParamsForTransport?.({

‎extensions/openrouter/openrouter.live.test.ts‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// Openrouter tests cover openrouter plugin behavior.
22
import OpenAI from "openai";
33
import { AuthStorage, ModelRegistry } from "openclaw/plugin-sdk/agent-sessions";
4+
import { streamSimple } from "openclaw/plugin-sdk/llm";
45
import {
56
registerProviderPlugin,
67
requireRegisteredProvider,
@@ -156,6 +157,51 @@ describeLive("openrouter plugin live", () => {
156157
await expectWeatherToolCall(client, autoModel.id);
157158
await expectWeatherToolCall(client, normalized.id);
158159
}, 30_000);
160+
161+
it("preserves a billed zero from the live free route", async () => {
162+
const { providers } = await registerOpenRouterPlugin();
163+
const provider = requireRegisteredProvider(providers, "openrouter");
164+
const resolved = provider.resolveDynamicModel?.({
165+
provider: "openrouter",
166+
modelId: "openrouter/free",
167+
modelRegistry: new ModelRegistryCtor(AuthStorage.inMemory()),
168+
});
169+
if (!resolved) {
170+
throw new Error("openrouter provider did not resolve openrouter/free");
171+
}
172+
const model =
173+
provider.normalizeResolvedModel?.({
174+
provider: "openrouter",
175+
modelId: resolved.id,
176+
model: resolved,
177+
}) ?? resolved;
178+
const wrapped = provider.wrapStreamFn?.({
179+
provider: "openrouter",
180+
modelId: model.id,
181+
streamFn: streamSimple,
182+
} as never);
183+
if (!wrapped) {
184+
throw new Error("openrouter provider did not expose its stream wrapper");
185+
}
186+
187+
const stream = await wrapped(
188+
model,
189+
{
190+
messages: [
191+
{
192+
role: "user",
193+
content: "Reply with exactly OK.",
194+
timestamp: Date.now(),
195+
},
196+
],
197+
},
198+
{ apiKey: OPENROUTER_API_KEY, maxTokens: 16 },
199+
);
200+
const message = await stream.result();
201+
202+
expect(message.usage.cost.total).toBe(0);
203+
expect(message.usage.cost.totalOrigin).toBe("provider-billed");
204+
}, 30_000);
159205
});
160206

161207
describeCatalogLive("openrouter plugin live model catalog", () => {

0 commit comments

Comments
 (0)