Skip to content

Commit bed36d8

Browse files
committed
fix(api,cost): hash long cache keys, clamp TTL splits, harden the TTL specs
What: - openai-native: hash over-long task ids with sha256 hex instead of slicing to 64 chars. A slice drops the distinguishing tail, so two ids sharing a 64-char prefix collapsed into one routing bucket. sha256 hex is exactly 64 chars, so the result still fits OpenAI's cap. - openai-native: clamp the 12h write subset at zero as well as at the write total. A negative `ttl_12h_tokens` from a misbehaving gateway used to reach the Task accumulator and the webview as a negative count. - cost: cap the 1h and 12h write subsets at the remaining aggregate write total. A malformed split (1h + 12h > total) billed the excess; 150k reported 1h writes against a 100k total cost 0.9 instead of 0.6. - Task.spec: replace the fixed 50ms sleep with `vi.waitFor` on the captured telemetry call. The drain is fire-and-forget and touches the filesystem, so a fixed sleep races it on a loaded worker. - vercel-ai-gateway.spec: assert the TTL argument of the `addCacheBreakpoints` mock call. The transform is file-mocked, so `JSON.stringify(mockCreate)` can never contain `cache_control` and the old assertion could not fail. - ttl.spec: compare raw `JSON.stringify` output against literal expected strings in the byte-identical test. `toEqual` on round-tripped objects ignores key insertion order, so a key reorder passed while the wire bytes changed. Why: Seven CodeRabbit "Minor" threads on the C5a/C5b stack. Three are production correctness (cache-key collision, negative token count, overbilled split); three are assertions that could not fail. The seventh thread's premise, forwarding the TTL split into the telemetry recompute, already landed in b647ec7. Impact: Long task ids keep distinct cache buckets; malformed TTL splits can no longer over- or under-bill or leak negative counts. No request shape changes for the default path: the aggregate-only case still derives both subsets as zero.
1 parent 65ac862 commit bed36d8

8 files changed

Lines changed: 128 additions & 57 deletions

File tree

‎src/api/providers/__tests__/openai-native-usage.spec.ts‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { describe, it, expect, beforeEach } from "vitest"
2+
import { createHash } from "crypto"
23
import { OpenAiNativeHandler } from "../openai-native"
34
import { OpenAiServiceTier, openAiNativeModels, type ModelInfo } from "@roo-code/types"
45
import type { ApiStreamUsageChunk } from "../../transform/stream"
@@ -495,7 +496,9 @@ describe("OpenAiNativeHandler - normalizeUsage", () => {
495496
const longTaskId = "t".repeat(80)
496497
const body = buildRequestBody("gpt-5.1", longTaskId)
497498
expect(body.prompt_cache_key).toHaveLength(64)
498-
expect(body.prompt_cache_key).toBe(longTaskId.slice(0, 64))
499+
// A hash, not a slice: truncation would collapse ids that share a prefix.
500+
expect(body.prompt_cache_key).toBe(createHash("sha256").update(longTaskId).digest("hex"))
501+
expect(body.prompt_cache_key).not.toBe(longTaskId.slice(0, 64))
499502
})
500503

501504
it("should keep prompt_cache_retention alongside the key for 24h models", () => {
@@ -761,5 +764,30 @@ describe("OpenAiNativeHandler - normalizeUsage", () => {
761764
// Total: 0.5 + 1.25 = 1.75
762765
expect(result.totalCost).toBeCloseTo(1.75, 6)
763766
})
767+
768+
it("should clamp a negative 12h split at zero instead of emitting a negative count", () => {
769+
// A misbehaving gateway can report a negative ttl_12h_tokens. Math.min alone
770+
// leaves it negative, so the negative count reaches the Task accumulator and
771+
// the webview. Shared costing clamps, but the emitted chunk must not lie.
772+
const model = getGpt6AstraModel()
773+
const usage = {
774+
input_tokens: 100_000,
775+
output_tokens: 0,
776+
input_tokens_details: {
777+
cache_write_tokens: 100_000,
778+
cache_write_tokens_details: {
779+
ttl_12h_tokens: -40_000,
780+
},
781+
},
782+
}
783+
784+
const result = normalizeUsage(handler, usage, model)
785+
786+
// A zero split is indistinguishable from "no split", so the field stays unset.
787+
expect(result.cacheWrite12hTokens).toBeUndefined()
788+
expect(result.cacheWrite12hTokens ?? 0).toBeGreaterThanOrEqual(0)
789+
// All 100k writes bill at the 30m rate: (12.5 / 1_000_000) * 100_000 = 1.25
790+
expect(result.totalCost).toBeCloseTo(1.25, 6)
791+
})
764792
})
765793
})

‎src/api/providers/__tests__/openai-native.spec.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1507,6 +1507,42 @@ describe("OpenAiNativeHandler", () => {
15071507
}
15081508
})
15091509

1510+
it("keeps two over-long task ids distinct instead of truncating them to one bucket", async () => {
1511+
// Task ids are uuidv7 today (36 chars), so this guards the wire contract for
1512+
// any caller that sends a longer id: a 64-char slice drops the distinguishing
1513+
// tail, so two ids sharing a 64-char prefix would land in one routing bucket.
1514+
const mockFetch = vitest.fn().mockResolvedValue({
1515+
ok: true,
1516+
body: new ReadableStream({
1517+
start(controller) {
1518+
controller.enqueue(new TextEncoder().encode("data: [DONE]\n\n"))
1519+
controller.close()
1520+
},
1521+
}),
1522+
})
1523+
global.fetch = mockFetch as typeof fetch
1524+
mockResponsesCreate.mockRejectedValue(new Error("SDK not available"))
1525+
1526+
const cacheHandler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" })
1527+
// 70 characters: identical for the first 64, different in the tail.
1528+
const sharedPrefix = "a".repeat(64)
1529+
const longTaskIds = [`${sharedPrefix}-one`, `${sharedPrefix}-two`]
1530+
1531+
for (const taskId of longTaskIds) {
1532+
for await (const _chunk of cacheHandler.createMessage(systemPrompt, messages, { taskId })) {
1533+
// consume
1534+
}
1535+
}
1536+
1537+
const bodies = mockFetch.mock.calls.map(([, init]) => JSON.parse(init.body))
1538+
expect(bodies).toHaveLength(2)
1539+
// OpenAI rejects keys over 64 characters, so the key must still fit.
1540+
expect(bodies[0].prompt_cache_key.length).toBeLessThanOrEqual(64)
1541+
expect(bodies[1].prompt_cache_key.length).toBeLessThanOrEqual(64)
1542+
// ...but the two tasks must not collide on that shortened form.
1543+
expect(bodies[1].prompt_cache_key).not.toBe(bodies[0].prompt_cache_key)
1544+
})
1545+
15101546
it("sends the task-scoped prompt_cache_key through the SDK path", async () => {
15111547
mockResponsesCreate.mockResolvedValue(asyncStreamFrom([]))
15121548

‎src/api/providers/__tests__/vercel-ai-gateway.spec.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -452,6 +452,7 @@ describe("VercelAiGatewayHandler", () => {
452452
it("does not emit a ttl for a gateway model with no TTL capability", async () => {
453453
// The gateway catalog is fetched at runtime and carries no TTL capability,
454454
// so a 1h knob must fail closed (ADR-0006 I2).
455+
const { addCacheBreakpoints } = await import("../../transform/caching/vercel-ai-gateway")
455456
const handler = new VercelAiGatewayHandler(
456457
makeApiHandlerOptions({
457458
...mockOptions,
@@ -462,8 +463,13 @@ describe("VercelAiGatewayHandler", () => {
462463

463464
await handler.createMessage("You are a helpful assistant.", [{ role: "user", content: "Hello" }]).next()
464465

465-
const call = mockCreate.mock.calls[mockCreate.mock.calls.length - 1][0]
466-
expect(JSON.stringify(call)).not.toContain("ttl")
466+
// `addCacheBreakpoints` is file-mocked, so the request body never receives
467+
// `cache_control` and `JSON.stringify(mockCreate...)` cannot see the TTL.
468+
// The TTL only exists as the third argument to the mocked call.
469+
expect(addCacheBreakpoints).toHaveBeenCalled()
470+
const lastCall = vitest.mocked(addCacheBreakpoints).mock.calls.at(-1)
471+
expect(lastCall).toBeDefined()
472+
expect(lastCall![2]).toBeUndefined()
467473
})
468474

469475
it("sets correct max_completion_tokens", async () => {

‎src/api/providers/openai-native.ts‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import * as os from "os"
2+
import { createHash } from "crypto"
23
import { v7 as uuidv7 } from "uuid"
34
import { Anthropic } from "@anthropic-ai/sdk"
45
import OpenAI from "openai"
@@ -162,8 +163,10 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
162163
usage.cache_read_input_tokens ?? usage.cache_read_tokens ?? usage.cached_tokens ?? cachedFromDetails ?? 0
163164

164165
// Clamp the 12h subset to the write total so a malformed response cannot bill
165-
// more 12h tokens than were written at all.
166-
const cacheWrite12hTokens = Math.min(writes12hFromDetails, cacheWriteTokens)
166+
// more 12h tokens than were written at all. The lower clamp matters too: a
167+
// negative `ttl_12h_tokens` from a misbehaving gateway would otherwise reach the
168+
// Task accumulator and the webview as a negative count.
169+
const cacheWrite12hTokens = Math.max(0, Math.min(writes12hFromDetails, cacheWriteTokens))
167170

168171
// Resolve effective tier: prefer actual tier from response; otherwise requested tier
169172
const effectiveTier =
@@ -1472,7 +1475,10 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
14721475
const key = metadata?.taskId || this.sessionId
14731476
if (!key) return undefined
14741477

1475-
return key.length > PROMPT_CACHE_KEY_MAX_LENGTH ? key.slice(0, PROMPT_CACHE_KEY_MAX_LENGTH) : key
1478+
// Hash rather than slice: a 64-char prefix drops the distinguishing tail, so two
1479+
// distinct task ids that share a long prefix would land in one routing bucket.
1480+
// sha256 hex is 64 characters, so the result fits the cap without truncation.
1481+
return key.length > PROMPT_CACHE_KEY_MAX_LENGTH ? createHash("sha256").update(key).digest("hex") : key
14761482
}
14771483

14781484
/**

‎src/api/transform/caching/__tests__/ttl.spec.ts‎

Lines changed: 10 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -116,51 +116,20 @@ describe("caching transforms under a prompt cache TTL", () => {
116116

117117
const vertexMessages = addVertexBreakpoints(buildAnthropicMessages())
118118

119-
expect(serialize(anthropicMessages)).toEqual(
120-
serialize([
121-
{
122-
role: "system",
123-
content: [{ type: "text", text: systemPrompt, cache_control: { type: "ephemeral" } }],
124-
},
125-
{
126-
role: "user",
127-
content: [{ type: "text", text: "First message", cache_control: { type: "ephemeral" } }],
128-
},
129-
{ role: "assistant", content: "First response" },
130-
{
131-
role: "user",
132-
content: [{ type: "text", text: "Second message", cache_control: { type: "ephemeral" } }],
133-
},
134-
]),
119+
// Compare the raw serialized bytes, not parsed objects: `toEqual` ignores key
120+
// insertion order, so a transform that reorders request keys would pass while
121+
// the bytes on the wire changed. The expected strings are literals for the
122+
// same reason: building them from the same helper would re-derive the order.
123+
expect(JSON.stringify(anthropicMessages)).toBe(
124+
'[{"role":"system","content":[{"type":"text","text":"You are a helpful assistant.","cache_control":{"type":"ephemeral"}}]},{"role":"user","content":[{"type":"text","text":"First message","cache_control":{"type":"ephemeral"}}]},{"role":"assistant","content":"First response"},{"role":"user","content":[{"type":"text","text":"Second message","cache_control":{"type":"ephemeral"}}]}]',
135125
)
136126

137-
expect(serialize(gatewayMessages)).toEqual(
138-
serialize([
139-
{ role: "system", content: systemPrompt, cache_control: { type: "ephemeral" } },
140-
{
141-
role: "user",
142-
content: [{ type: "text", text: "First message", cache_control: { type: "ephemeral" } }],
143-
},
144-
{ role: "assistant", content: "First response" },
145-
{
146-
role: "user",
147-
content: [{ type: "text", text: "Second message", cache_control: { type: "ephemeral" } }],
148-
},
149-
]),
127+
expect(JSON.stringify(gatewayMessages)).toBe(
128+
'[{"role":"system","content":"You are a helpful assistant.","cache_control":{"type":"ephemeral"}},{"role":"user","content":[{"type":"text","text":"First message","cache_control":{"type":"ephemeral"}}]},{"role":"assistant","content":"First response"},{"role":"user","content":[{"type":"text","text":"Second message","cache_control":{"type":"ephemeral"}}]}]',
150129
)
151130

152-
expect(serialize(vertexMessages)).toEqual(
153-
serialize([
154-
{
155-
role: "user",
156-
content: [{ type: "text", text: "First message", cache_control: { type: "ephemeral" } }],
157-
},
158-
{ role: "assistant", content: "First response" },
159-
{
160-
role: "user",
161-
content: [{ type: "text", text: "Second message", cache_control: { type: "ephemeral" } }],
162-
},
163-
]),
131+
expect(JSON.stringify(vertexMessages)).toBe(
132+
'[{"role":"user","content":[{"type":"text","text":"First message","cache_control":{"type":"ephemeral"}}]},{"role":"assistant","content":"First response"},{"role":"user","content":[{"type":"text","text":"Second message","cache_control":{"type":"ephemeral"}}]}]',
164133
)
165134
})
166135
})

‎src/core/task/__tests__/Task.spec.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -567,8 +567,14 @@ describe("Cline", () => {
567567

568568
await task.recursivelyMakeClineRequests([{ type: "text", text: "original user request" }])
569569

570-
// Allow the fire-and-forget drain to settle.
571-
await new Promise((resolve) => setTimeout(resolve, 50))
570+
// The drain is fire-and-forget and reaches `captureLlmCompletion` after several
571+
// awaits, some of which touch the filesystem. A fixed sleep races that work on a
572+
// loaded worker, so poll for the captured call instead.
573+
await vi.waitFor(() =>
574+
expect(
575+
captureLlmCompletion.mock.calls.some(([, payload]) => payload?.cacheWriteTokens === 100_000),
576+
).toBe(true),
577+
)
572578

573579
const call = captureLlmCompletion.mock.calls.find(([, payload]) => payload?.cacheWriteTokens === 100_000)
574580
expect(call).toBeDefined()

‎src/shared/cost.ts‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -86,9 +86,12 @@ function calculateApiCostInternal(
8686
// The per-TTL counts are a decomposition of `cacheCreationInputTokens`, never
8787
// an addition to it. Deriving the 5m remainder this way keeps the aggregate
8888
// path byte-identical when a vendor reports no split (both terms are 0).
89-
const writes1h = Math.max(0, cacheCreation1hInputTokens)
90-
const writes12h = Math.max(0, cacheCreation12hInputTokens)
91-
const writes5m = Math.max(0, cacheCreationInputTokens - writes1h - writes12h)
89+
// Each subset also caps at what the aggregate has left, so a malformed split
90+
// (1h + 12h > total) cannot bill more writes than the vendor reported.
91+
const writesTotal = Math.max(0, cacheCreationInputTokens)
92+
const writes1h = Math.min(Math.max(0, cacheCreation1hInputTokens), writesTotal)
93+
const writes12h = Math.min(Math.max(0, cacheCreation12hInputTokens), writesTotal - writes1h)
94+
const writes5m = Math.max(0, writesTotal - writes1h - writes12h)
9295

9396
const writes5mRate = modelInfo.cacheWritesPrice || 0
9497
const writes1hRate = resolveCacheWriteRate(modelInfo, modelInfo.cacheWritesPrice1h, CACHE_WRITE_MULTIPLIER_1H) || 0

‎src/utils/__tests__/cost.spec.ts‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -358,15 +358,32 @@ describe("Cost Utility", () => {
358358
expect(result.totalCost).toBeCloseTo(0.5175, 9)
359359
})
360360

361-
it("clamps the 5m remainder at zero when the split exceeds the total", () => {
361+
it("caps each TTL class at the aggregate write total", () => {
362362
// A malformed response claiming more 1h tokens than the write total must
363-
// not produce negative 5m writes (which would refund cost).
363+
// not bill the excess. The 5m remainder already clamps at zero, but the 1h
364+
// and 12h classes were billed at their full reported count.
364365
const result = calculateApiCostAnthropic(anthropicModel, 0, 0, 100_000, 0, 150_000)
365366

366-
// 5m remainder clamps to 0; 1h is capped by the clamp on the remainder only,
367-
// so the 1h class still bills its reported 150k at 6.0 => 0.9
368-
expect(result.totalCost).toBeCloseTo(0.9, 9)
367+
// 1h writes cap at the 100k aggregate; the 5m remainder clamps to 0.
368+
// 100k at 6.0 => 0.6
369+
expect(result.totalCost).toBeCloseTo(0.6, 9)
369370
expect(result.totalCost).toBeGreaterThan(0)
371+
// The pre-fix behaviour billed all 150k at 6.0 => 0.9.
372+
expect(result.totalCost).not.toBeCloseTo(0.9, 9)
373+
})
374+
375+
it("caps the 12h class at the remainder left after the 1h class", () => {
376+
// 1h and 12h are a decomposition of one aggregate, so they cannot both
377+
// claim the whole total. 1h wins the ordering; 12h gets what is left.
378+
const model: ModelInfo = { ...anthropicModel, cacheWritesPrice12h: 7.5 }
379+
const result = calculateApiCostAnthropic(model, 0, 0, 100_000, 0, 60_000, 80_000)
380+
381+
// 1h: 60k at 6.0 => 0.36
382+
// 12h: capped at 100k - 60k = 40k at 7.5 => 0.3
383+
// 5m: 100k - 60k - 40k = 0
384+
// Total: 0.66 (the pre-fix behaviour billed 80k 12h => 0.36 + 0.6 = 0.96)
385+
expect(result.totalCost).toBeCloseTo(0.66, 9)
386+
expect(result.totalCost).not.toBeCloseTo(0.96, 9)
370387
})
371388

372389
it("is byte-identical to the pre-split cost when no split is reported", () => {

0 commit comments

Comments
 (0)