Skip to content

Commit 599e036

Browse files
committed
Count cache writes, not just cache hits
Preparation for pointing cxagent at OpenRouter. The wiring already exists — it is a first-class preset with base URL, attribution header, Bearer auth and cached_tokens parsing — so a smoke run needs no new code. This is the one number that would have been wrong. A HIT IS NOT ALWAYS FREE. On a local llama.cpp, filling the cache costs nothing: it is the server's own RAM, and it reports no writes at all. Through OpenRouter it is billed — OpenAI charges 1.25x normal input to write, Anthropic 1.25x for a five-minute entry and 2x for an hour. So a session can show a healthy hit rate while the warming cost more than the reads saved, and the two cases are indistinguishable from the hit rate alone. Reporting hits without writes is worse than reporting neither, because the number looks trustworthy. opencode carries this exact bug open (anomalyco/opencode#18440): cache write tokens unaccounted, costs understated. prompt_tokens_details.cache_write_tokens now flows through LlmUsage, the ledger, the panel and /stats, with a schema migration for history. A write with no read still counts as REPORTED: the first turn against a paid provider warms the cache and hits nothing, which is a real measurement of an expensive turn rather than a silent provider. Both readouts stay a single clean figure where writes are zero, so nothing changes on a local endpoint — verified: 47 history rows migrated intact, writes 0 throughout.
1 parent ab7c1c5 commit 599e036

12 files changed

Lines changed: 184 additions & 16 deletions

File tree

‎cxagent.Tests/OpenAiCompatibleProviderTests.cs‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,66 @@ public async Task ChatStreamAsync_ReadsCachedPromptTokens()
8181
Assert.True(usage.CacheReported);
8282
}
8383

84+
/// <summary>
85+
/// CACHE WRITES, WHERE THEY ARE BILLED. OpenRouter reports cache_write_tokens alongside the
86+
/// reads; a local llama.cpp reports neither, because filling its own RAM is free. Reading hits
87+
/// without writes understates cost on exactly the providers that charge for warming — OpenAI at
88+
/// 1.25x normal input, Anthropic up to 2x — and a number that looks trustworthy while being
89+
/// wrong is worse than no number.
90+
/// </summary>
91+
[Fact]
92+
public async Task ChatAsync_ReadsCacheWriteTokens()
93+
{
94+
_srv.EnqueueJson(200, """
95+
{"choices":[{"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],
96+
"usage":{"prompt_tokens":2000,"completion_tokens":3,
97+
"prompt_tokens_details":{"cached_tokens":500,"cache_write_tokens":1500}}}
98+
""");
99+
100+
var r = await Make().ChatAsync(Msgs(), null, CancellationToken.None);
101+
102+
Assert.Equal(500, r.Usage.CachedInputTokens);
103+
Assert.Equal(1500, r.Usage.CacheWriteTokens);
104+
Assert.True(r.Usage.CacheReported);
105+
}
106+
107+
/// <summary>
108+
/// A WRITE WITH NO READ STILL COUNTS AS REPORTED. The first turn against a paid provider warms
109+
/// the cache and hits nothing — that is a real measurement of an expensive turn, not a silent
110+
/// provider, and treating it as unreported would hide the cost entirely.
111+
/// </summary>
112+
[Fact]
113+
public async Task ChatAsync_AWriteWithoutAHit_IsStillReported()
114+
{
115+
_srv.EnqueueJson(200, """
116+
{"choices":[{"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],
117+
"usage":{"prompt_tokens":2000,"completion_tokens":3,
118+
"prompt_tokens_details":{"cache_write_tokens":2000}}}
119+
""");
120+
121+
var r = await Make().ChatAsync(Msgs(), null, CancellationToken.None);
122+
123+
Assert.True(r.Usage.CacheReported);
124+
Assert.Equal(2000, r.Usage.CacheWriteTokens);
125+
Assert.Equal(0, r.Usage.CachedInputTokens);
126+
}
127+
128+
/// <summary>A local endpoint sends no cache block at all — writes stay zero and nothing claims
129+
/// a cost that was never incurred.</summary>
130+
[Fact]
131+
public async Task ChatAsync_WithNoCacheBlock_ReportsNoWrites()
132+
{
133+
_srv.EnqueueJson(200, """
134+
{"choices":[{"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],
135+
"usage":{"prompt_tokens":1000,"completion_tokens":3}}
136+
""");
137+
138+
var r = await Make().ChatAsync(Msgs(), null, CancellationToken.None);
139+
140+
Assert.False(r.Usage.CacheReported);
141+
Assert.Equal(0, r.Usage.CacheWriteTokens);
142+
}
143+
84144
[Fact]
85145
public async Task ChatAsync_MapsTextResponse_AndNormalizesStopReason()
86146
{

‎cxagent.Tests/TokenLedgerTests.cs‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,33 @@ public void WithNothingReported_BothSidesAreNull()
120120
Assert.Null(workers);
121121
}
122122

123+
/// <summary>
124+
/// THE LEDGER CARRIES WHAT THE WARMING COST. A session can show a healthy hit rate while the
125+
/// writes that produced it cost more than the reads saved — true on OpenAI and Anthropic, never
126+
/// true on a local endpoint, and indistinguishable without this figure.
127+
/// </summary>
128+
[Fact]
129+
public void Record_AccumulatesCacheWrites()
130+
{
131+
var l = new TokenLedger();
132+
l.Record(new LlmUsage { InputTokens = 2000, OutputTokens = 5, CacheWriteTokens = 2000, CacheReported = true });
133+
l.Record(new LlmUsage { InputTokens = 2000, OutputTokens = 5, CachedInputTokens = 1900, CacheReported = true });
134+
135+
Assert.Equal(2000, l.CacheWrittenTokens);
136+
Assert.Equal(1900, l.CachedInputTokens);
137+
}
138+
139+
/// <summary>Zero where warming is free — every local endpoint — so the readout stays a single
140+
/// clean figure rather than claiming a cost that was never incurred.</summary>
141+
[Fact]
142+
public void WithNoWritesReported_TheWriteTallyStaysZero()
143+
{
144+
var l = new TokenLedger();
145+
l.Record(new LlmUsage { InputTokens = 1000, OutputTokens = 5, CachedInputTokens = 900, CacheReported = true });
146+
147+
Assert.Equal(0, l.CacheWrittenTokens);
148+
}
149+
123150
// ---- concurrency ---------------------------------------------------------------------------
124151

125152
/// <summary>

‎cxagent/Core/Agent/AgentHost.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -614,7 +614,8 @@ private Agent BuildAgent()
614614
agent.Id, _runtime.WorkingDir, SpendLabel, Mode.ToString(),
615615
Ledger.InputTokens, Ledger.OutputTokens, Ledger.SubAgentTokens, _turns,
616616
_startedAt, DateTimeOffset.UtcNow,
617-
Ledger.CachedInputTokens, Ledger.CacheHitRate is not null));
617+
Ledger.CachedInputTokens, Ledger.CacheWrittenTokens,
618+
Ledger.CacheHitRate is not null));
618619
};
619620

620621
// OCCUPANCY, which nothing else in this mode observes. Without it the status bar has only the

‎cxagent/Core/Llm/LlmTypes.cs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,24 @@ public record LlmUsage
2828
/// <summary>Whether the provider reported cache figures at all — see
2929
/// <see cref="CachedInputTokens"/> for why the distinction matters.</summary>
3030
public bool CacheReported { get; init; }
31+
32+
/// <summary>
33+
/// Input tokens WRITTEN to the provider's cache on this call.
34+
///
35+
/// <para>A HIT IS NOT ALWAYS FREE, AND A WRITE IS SOMETIMES EXPENSIVE. On a local llama.cpp the
36+
/// cache costs nothing to fill — it is the server's own RAM. Through OpenRouter it is billed:
37+
/// OpenAI charges 1.25x normal input to write, Anthropic 1.25x for a 5-minute entry and 2x for a
38+
/// one-hour one. So a session can show a healthy hit rate while the warming quietly cost more
39+
/// than the reads saved.</para>
40+
///
41+
/// <para>REPORTING HITS WITHOUT WRITES IS WORSE THAN REPORTING NEITHER, because the number
42+
/// looks trustworthy. opencode carries this exact bug open (anomalyco/opencode#18440): cache
43+
/// write tokens unaccounted, costs understated.</para>
44+
///
45+
/// <para>Zero on a provider that never writes, and on every local endpoint — which is why it is
46+
/// counted separately rather than folded into <see cref="CachedInputTokens"/>.</para>
47+
/// </summary>
48+
public int CacheWriteTokens { get; init; }
3149
}
3250

3351
public record LlmResponse

‎cxagent/Core/Llm/Providers/OpenAiWire.cs‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -292,21 +292,36 @@ public static bool TryParseUsageOnlyChunk(string dataJson, out LlmUsage usage)
292292
private static LlmUsage ReadUsage(JsonElement u)
293293
{
294294
var cached = 0;
295+
var written = 0;
295296
var reported = false;
296297
if (u.TryGetProperty("prompt_tokens_details", out var details)
297-
&& details.ValueKind == JsonValueKind.Object
298-
&& details.TryGetProperty("cached_tokens", out var ctok)
299-
&& ctok.ValueKind == JsonValueKind.Number)
298+
&& details.ValueKind == JsonValueKind.Object)
300299
{
301-
cached = ctok.GetInt32();
302-
reported = true;
300+
if (details.TryGetProperty("cached_tokens", out var ctok)
301+
&& ctok.ValueKind == JsonValueKind.Number)
302+
{
303+
cached = ctok.GetInt32();
304+
reported = true;
305+
}
306+
307+
// WRITES, WHERE THEY ARE BILLED. OpenRouter reports this alongside the reads; a local
308+
// llama.cpp does not send it at all, which is correct — filling its own RAM is free.
309+
// A provider that reports writes but no reads still counts as reporting: the absence of
310+
// hits on a turn that warmed the cache is a real measurement, not a silent provider.
311+
if (details.TryGetProperty("cache_write_tokens", out var wtok)
312+
&& wtok.ValueKind == JsonValueKind.Number)
313+
{
314+
written = wtok.GetInt32();
315+
reported = true;
316+
}
303317
}
304318

305319
return new LlmUsage
306320
{
307321
InputTokens = u.TryGetProperty("prompt_tokens", out var pt) ? pt.GetInt32() : 0,
308322
OutputTokens = u.TryGetProperty("completion_tokens", out var ct) ? ct.GetInt32() : 0,
309323
CachedInputTokens = cached,
324+
CacheWriteTokens = written,
310325
CacheReported = reported,
311326
};
312327
}

‎cxagent/Core/Llm/TokenLedger.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ public sealed class TokenLedger
2121
private int _input;
2222
private int _output;
2323
private int _cached;
24+
private int _cacheWritten;
2425
private int _cacheReports;
2526

2627
// THE SAME FOUR NUMBERS, SPLIT BY WHO SPENT THEM. See CacheHitRateByAgent for why the session
@@ -71,6 +72,16 @@ public TokenLedger(int inputTokens, int outputTokens)
7172
/// </summary>
7273
public int CachedInputTokens => Volatile.Read(ref _cached);
7374

75+
/// <summary>
76+
/// Input tokens written INTO the provider's cache, cumulatively.
77+
///
78+
/// <para>ZERO EVERYWHERE THE CACHE IS FREE — a local endpoint fills its own RAM and reports
79+
/// nothing. Non-zero only on a provider that BILLS for warming, where OpenAI charges 1.25x
80+
/// normal input and Anthropic up to 2x. A hit rate shown without this reads as pure saving on
81+
/// exactly the providers where it is not.</para>
82+
/// </summary>
83+
public int CacheWrittenTokens => Volatile.Read(ref _cacheWritten);
84+
7485
/// <summary>
7586
/// The share of input served from cache, or null when no provider ever reported the figure.
7687
///
@@ -220,6 +231,7 @@ public void Record(LlmUsage usage, string? modelId = null, bool subAgent = false
220231
if (usage.CacheReported)
221232
{
222233
Interlocked.Add(ref _cached, usage.CachedInputTokens);
234+
Interlocked.Add(ref _cacheWritten, usage.CacheWriteTokens);
223235
Interlocked.Increment(ref _cacheReports);
224236
}
225237
Interlocked.Add(ref _output, usage.OutputTokens);

‎cxagent/Core/Storage/StatsQuery.cs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@ namespace CxAgent.Core.Storage;
33
/// <summary>Totals for one window of time.</summary>
44
public sealed record StatsTotals(
55
int Sessions, int InputTokens, int OutputTokens, int SubAgentTokens, int Turns,
6-
int CachedInputTokens = 0, int CacheReportingInputTokens = 0)
6+
int CachedInputTokens = 0, int CacheReportingInputTokens = 0,
7+
int CacheWrittenTokens = 0)
78
{
89
public int TotalTokens => InputTokens + OutputTokens;
910

@@ -76,7 +77,8 @@ public static StatsTotals Totals(IReadOnlyList<SessionRecord> sessions) =>
7677
sessions.Sum(s => s.SubAgentTokens),
7778
sessions.Sum(s => s.Turns),
7879
sessions.Sum(s => s.CachedInputTokens),
79-
sessions.Where(s => s.CacheReported).Sum(s => s.InputTokens));
80+
sessions.Where(s => s.CacheReported).Sum(s => s.InputTokens),
81+
sessions.Sum(s => s.CacheWrittenTokens));
8082

8183
/// <summary>Busiest project first. Unattributed sessions are grouped rather than dropped: a
8284
/// session with no working directory is still spend, and hiding it would make the parts stop

‎cxagent/Core/Storage/UsageHistoryStore.cs‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@ public sealed record SessionRecord(
99
DateTimeOffset StartedAt, DateTimeOffset UpdatedAt,
1010
/// <summary>Input tokens the provider served from its prefix cache.</summary>
1111
int CachedInputTokens = 0,
12+
/// <summary>Input tokens written INTO the provider's cache — zero where warming is free, which
13+
/// is every local endpoint. Non-zero only where it is BILLED, and a hit rate reported without
14+
/// it reads as pure saving on exactly those providers.</summary>
15+
int CacheWrittenTokens = 0,
1216
/// <summary>Whether the provider reported cache figures at all. False makes
1317
/// <see cref="CachedInputTokens"/> mean "unknown" rather than "none", which is the difference
1418
/// between staying silent and claiming a 0% hit rate.</summary>
@@ -168,6 +172,7 @@ CREATE TABLE IF NOT EXISTS permissions (
168172
// the developer's own.
169173
AddColumnIfMissing(conn, "sessions", "cached_input_tokens", "INTEGER NOT NULL DEFAULT 0");
170174
AddColumnIfMissing(conn, "sessions", "cache_reported", "INTEGER NOT NULL DEFAULT 0");
175+
AddColumnIfMissing(conn, "sessions", "cache_written_tokens", "INTEGER NOT NULL DEFAULT 0");
171176
}
172177
catch (Exception)
173178
{
@@ -215,9 +220,9 @@ public void SaveSession(SessionRecord r)
215220
INSERT INTO sessions
216221
(agent_id, working_dir, model_id, mode, input_tokens, output_tokens,
217222
sub_agent_tokens, turns, started_at, updated_at,
218-
cached_input_tokens, cache_reported)
223+
cached_input_tokens, cache_reported, cache_written_tokens)
219224
VALUES ($id, $dir, $model, $mode, $in, $out, $sub, $turns, $started, $updated,
220-
$cached, $creported)
225+
$cached, $creported, $cwritten)
221226
ON CONFLICT(agent_id) DO UPDATE SET
222227
working_dir = excluded.working_dir,
223228
model_id = excluded.model_id,
@@ -228,7 +233,8 @@ ON CONFLICT(agent_id) DO UPDATE SET
228233
turns = excluded.turns,
229234
updated_at = excluded.updated_at,
230235
cached_input_tokens = excluded.cached_input_tokens,
231-
cache_reported = excluded.cache_reported;
236+
cache_reported = excluded.cache_reported,
237+
cache_written_tokens = excluded.cache_written_tokens;
232238
""";
233239
cmd.Parameters.AddWithValue("$id", r.AgentId);
234240
cmd.Parameters.AddWithValue("$dir", (object?)r.WorkingDir ?? DBNull.Value);
@@ -242,6 +248,7 @@ ON CONFLICT(agent_id) DO UPDATE SET
242248
cmd.Parameters.AddWithValue("$updated", Stamp(r.UpdatedAt));
243249
cmd.Parameters.AddWithValue("$cached", r.CachedInputTokens);
244250
cmd.Parameters.AddWithValue("$creported", r.CacheReported ? 1 : 0);
251+
cmd.Parameters.AddWithValue("$cwritten", r.CacheWrittenTokens);
245252
cmd.ExecuteNonQuery();
246253
}
247254
catch (Exception) { }
@@ -408,7 +415,7 @@ public IReadOnlyList<SessionRecord> SessionsSince(DateTimeOffset since)
408415
cmd.CommandText = """
409416
SELECT agent_id, working_dir, model_id, mode, input_tokens, output_tokens,
410417
sub_agent_tokens, turns, started_at, updated_at,
411-
cached_input_tokens, cache_reported
418+
cached_input_tokens, cache_reported, cache_written_tokens
412419
FROM sessions WHERE updated_at >= $since ORDER BY updated_at DESC;
413420
""";
414421
cmd.Parameters.AddWithValue("$since", Stamp(since));
@@ -424,7 +431,7 @@ public IReadOnlyList<SessionRecord> SessionsSince(DateTimeOffset since)
424431
reader.GetInt32(4), reader.GetInt32(5), reader.GetInt32(6), reader.GetInt32(7),
425432
DateTimeOffset.Parse(reader.GetString(8), System.Globalization.CultureInfo.InvariantCulture),
426433
DateTimeOffset.Parse(reader.GetString(9), System.Globalization.CultureInfo.InvariantCulture),
427-
reader.GetInt32(10), reader.GetInt32(11) != 0));
434+
reader.GetInt32(10), reader.GetInt32(12), reader.GetInt32(11) != 0));
428435
return list;
429436
}
430437

‎cxagent/UI/AppBootstrap.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -480,7 +480,7 @@ void WireRunner(ProviderResolution res)
480480
// panel is everything, and "Tokens by agent" is where the two are reconciled.
481481
mainWindow.SetSpendByModel(runner.Ledger.ByModel, runner.Ledger.SubAgentTokens,
482482
runner.Ledger.SplitByModel, runner.Ledger.CacheHitRate,
483-
runner.Ledger.CacheHitRateByAgent);
483+
runner.Ledger.CacheHitRateByAgent, runner.Ledger.CacheWrittenTokens);
484484
});
485485
runner.ContextUsedUpdated += (_, used) => system.EnqueueOnUIThread(() => mainWindow.SetContextUsed(used));
486486
runner.ContextCompressed += (_, d) => system.EnqueueOnUIThread(() => mainWindow.MarkContextStale(d.Before, d.After));

‎cxagent/UI/MainWindow.cs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -341,7 +341,8 @@ public string StartupMode
341341
public void SetSpendByModel(IReadOnlyDictionary<string, int> byModel, int subAgentTokens = 0,
342342
IReadOnlyDictionary<string, (int Input, int Output)>? splitByModel = null,
343343
double? cacheHitRate = null,
344-
(double? Own, double? Workers) cacheByAgent = default)
344+
(double? Own, double? Workers) cacheByAgent = default,
345+
int cacheWritten = 0)
345346
{
346347
_spendByModel = byModel;
347348
_subAgentTokens = subAgentTokens;
@@ -353,12 +354,16 @@ public void SetSpendByModel(IReadOnlyDictionary<string, int> byModel, int subAge
353354
_cacheHitRate = cacheHitRate;
354355
_ownCacheHitRate = cacheByAgent.Own;
355356
_workerCacheHitRate = cacheByAgent.Workers;
357+
_cacheWritten = cacheWritten;
356358
RefreshSessionPanel();
357359
}
358360

359361
/// <summary>Share of input served from the provider's prefix cache; null when unreported.</summary>
360362
private double? _cacheHitRate;
361363

364+
/// <summary>Input tokens written into the provider's cache; zero where warming is free.</summary>
365+
private int _cacheWritten;
366+
362367
/// <summary>The same, split by who spent it — see TokenLedger.CacheHitRateByAgent.</summary>
363368
private double? _ownCacheHitRate;
364369
private double? _workerCacheHitRate;
@@ -1086,6 +1091,7 @@ public void RefreshSessionPanel()
10861091
SpendByModel = _spendByModel,
10871092
SplitByModel = _splitByModel,
10881093
CacheHitRate = _cacheHitRate,
1094+
CacheWrittenTokens = _cacheWritten,
10891095
OwnCacheHitRate = _ownCacheHitRate,
10901096
WorkerCacheHitRate = _workerCacheHitRate,
10911097

0 commit comments

Comments
 (0)