Skip to content

Commit 5123cf4

Browse files
halindromeclaude
andcommitted
fix(metrics): address QA round 1 findings
- Fix #1: EstimateTokens comment says "bytes" not "characters" (accurate) - Fix #3: Unexport Tracker fields (InstallID, TotalTokensSaved, TotalCostAvoided) to prevent unsynchronized access; use Snapshot() instead - Fix #6: Add priceForConfig test covering all 4 pricing branches - Fix #7: Add initMetricsTracker test (default enabled + disabled via config) - Fix DeusData#8: Rename CompressionRatio → ReductionRatio (ratio < 1 = reduction, > 1 = expansion — semantically clearer) - Fix DeusData#10: Add GetFloat64 unit test for config store - Design choice comment: document which tools are instrumented and why others are intentionally excluded from _meta Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
1 parent 9528dd9 commit 5123cf4

6 files changed

Lines changed: 162 additions & 30 deletions

File tree

‎internal/metrics/savings.go‎

Lines changed: 19 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,11 @@ type TokenMetadata struct {
1919
BaselineTokens int `json:"baseline_tokens"`
2020
ResponseTokens int `json:"response_tokens"`
2121
CostAvoided float64 `json:"cost_avoided"`
22-
CompressionRatio float64 `json:"compression_ratio"`
22+
ReductionRatio float64 `json:"reduction_ratio"`
2323
}
2424

2525
// EstimateTokens approximates token count from byte length using
26-
// the standard English heuristic of 1 token ≈ 4 characters.
26+
// the heuristic of 1 token ≈ 4 bytes (accurate for ASCII-heavy source code).
2727
func EstimateTokens(s string) int {
2828
return len(s) / 4
2929
}
@@ -48,7 +48,7 @@ func CalculateSavings(baselineBytes, responseBytes int, pricePerToken float64) T
4848
BaselineTokens: baseline,
4949
ResponseTokens: response,
5050
CostAvoided: math.Round(float64(saved)*pricePerToken*1e6) / 1e6,
51-
CompressionRatio: ratio,
51+
ReductionRatio: ratio,
5252
}
5353
}
5454

@@ -61,12 +61,13 @@ type savingsRecord struct {
6161
}
6262

6363
// Tracker accumulates token savings across calls and persists totals.
64+
// Use Snapshot() to read totals safely — do not access fields directly.
6465
type Tracker struct {
6566
mu sync.Mutex
6667
path string
67-
InstallID string
68-
TotalTokensSaved int64
69-
TotalCostAvoided float64
68+
installID string
69+
totalTokensSaved int64
70+
totalCostAvoided float64
7071
}
7172

7273
// NewTracker loads or creates savings.json at path. Generates a random
@@ -79,20 +80,20 @@ func NewTracker(path string) *Tracker {
7980
if !os.IsNotExist(err) {
8081
slog.Warn("metrics: failed to read savings file", "path", path, "err", err)
8182
}
82-
t.InstallID = newInstallID()
83+
t.installID = newInstallID()
8384
return t
8485
}
8586

8687
var rec savingsRecord
8788
if err := json.Unmarshal(data, &rec); err != nil {
8889
slog.Warn("metrics: malformed savings file, starting fresh", "path", path, "err", err)
89-
t.InstallID = newInstallID()
90+
t.installID = newInstallID()
9091
return t
9192
}
9293

93-
t.InstallID = rec.InstallID
94-
t.TotalTokensSaved = rec.TotalTokensSaved
95-
t.TotalCostAvoided = rec.TotalCostAvoided
94+
t.installID = rec.InstallID
95+
t.totalTokensSaved = rec.TotalTokensSaved
96+
t.totalCostAvoided = rec.TotalCostAvoided
9697
return t
9798
}
9899

@@ -102,13 +103,13 @@ func (t *Tracker) Record(meta TokenMetadata) {
102103
t.mu.Lock()
103104
defer t.mu.Unlock()
104105

105-
t.TotalTokensSaved += int64(meta.TokensSaved)
106-
t.TotalCostAvoided += meta.CostAvoided
106+
t.totalTokensSaved += int64(meta.TokensSaved)
107+
t.totalCostAvoided += meta.CostAvoided
107108

108109
rec := savingsRecord{
109-
InstallID: t.InstallID,
110-
TotalTokensSaved: t.TotalTokensSaved,
111-
TotalCostAvoided: t.TotalCostAvoided,
110+
InstallID: t.installID,
111+
TotalTokensSaved: t.totalTokensSaved,
112+
TotalCostAvoided: t.totalCostAvoided,
112113
LastUpdated: time.Now().UTC().Format(time.RFC3339),
113114
}
114115

@@ -144,14 +145,14 @@ func (t *Tracker) Record(meta TokenMetadata) {
144145
return
145146
}
146147

147-
slog.Debug("metrics: savings persisted", "path", t.path, "total_tokens_saved", t.TotalTokensSaved)
148+
slog.Debug("metrics: savings persisted", "path", t.path, "total_tokens_saved", t.totalTokensSaved)
148149
}
149150

150151
// Snapshot returns current cumulative totals under lock.
151152
func (t *Tracker) Snapshot() (totalTokensSaved int64, totalCostAvoided float64) {
152153
t.mu.Lock()
153154
defer t.mu.Unlock()
154-
return t.TotalTokensSaved, t.TotalCostAvoided
155+
return t.totalTokensSaved, t.totalCostAvoided
155156
}
156157

157158
func newInstallID() string {

‎internal/metrics/savings_test.go‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -23,39 +23,39 @@ func TestCalculateSavings(t *testing.T) {
2323
responseBytes int
2424
pricePerToken float64
2525
wantTokensSaved int
26-
wantCompressionRatio float64
26+
wantReductionRatio float64
2727
}{
2828
{
2929
name: "NormalCase",
3030
baselineBytes: 4000,
3131
responseBytes: 400,
3232
pricePerToken: 0.000015,
3333
wantTokensSaved: 900,
34-
wantCompressionRatio: 0.1,
34+
wantReductionRatio: 0.1,
3535
},
3636
{
3737
name: "NoSavings",
3838
baselineBytes: 100,
3939
responseBytes: 200,
4040
pricePerToken: 0.000015,
4141
wantTokensSaved: 0,
42-
wantCompressionRatio: 2.0,
42+
wantReductionRatio: 2.0,
4343
},
4444
{
4545
name: "ZeroBaseline",
4646
baselineBytes: 0,
4747
responseBytes: 400,
4848
pricePerToken: 0.000015,
4949
wantTokensSaved: 0,
50-
wantCompressionRatio: 0.0,
50+
wantReductionRatio: 0.0,
5151
},
5252
{
5353
name: "ZeroBoth",
5454
baselineBytes: 0,
5555
responseBytes: 0,
5656
pricePerToken: 0.000015,
5757
wantTokensSaved: 0,
58-
wantCompressionRatio: 0.0,
58+
wantReductionRatio: 0.0,
5959
},
6060
}
6161

@@ -65,8 +65,8 @@ func TestCalculateSavings(t *testing.T) {
6565
if got.TokensSaved != tt.wantTokensSaved {
6666
t.Errorf("TokensSaved = %d, want %d", got.TokensSaved, tt.wantTokensSaved)
6767
}
68-
if got.CompressionRatio != tt.wantCompressionRatio {
69-
t.Errorf("CompressionRatio = %v, want %v", got.CompressionRatio, tt.wantCompressionRatio)
68+
if got.ReductionRatio != tt.wantReductionRatio {
69+
t.Errorf("ReductionRatio = %v, want %v", got.ReductionRatio, tt.wantReductionRatio)
7070
}
7171
})
7272
}
@@ -137,8 +137,8 @@ func TestTracker_LoadExisting(t *testing.T) {
137137
}
138138

139139
tr := NewTracker(path)
140-
if tr.InstallID != "abc123" {
141-
t.Errorf("InstallID = %q, want %q", tr.InstallID, "abc123")
140+
if tr.installID != "abc123" {
141+
t.Errorf("InstallID = %q, want %q", tr.installID, "abc123")
142142
}
143143

144144
tr.Record(TokenMetadata{TokensSaved: 200})

‎internal/store/config_test.go‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package store
2+
3+
import (
4+
"testing"
5+
)
6+
7+
func TestGetFloat64(t *testing.T) {
8+
cfg, err := OpenConfigInDir(t.TempDir())
9+
if err != nil {
10+
t.Fatal(err)
11+
}
12+
defer cfg.Close()
13+
14+
// Default when key is missing.
15+
if got := cfg.GetFloat64("missing_key", 3.14); got != 3.14 {
16+
t.Errorf("GetFloat64 missing key = %v, want 3.14", got)
17+
}
18+
19+
// Valid float value.
20+
if err := cfg.Set("price", "0.000015"); err != nil {
21+
t.Fatal(err)
22+
}
23+
if got := cfg.GetFloat64("price", 0); got != 0.000015 {
24+
t.Errorf("GetFloat64 valid = %v, want 0.000015", got)
25+
}
26+
27+
// Invalid value falls back to default.
28+
if err := cfg.Set("bad", "not-a-number"); err != nil {
29+
t.Fatal(err)
30+
}
31+
if got := cfg.GetFloat64("bad", 99.9); got != 99.9 {
32+
t.Errorf("GetFloat64 invalid = %v, want 99.9", got)
33+
}
34+
}

‎internal/tools/snippet_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -516,7 +516,7 @@ func TestGetCodeSnippet_MetaField(t *testing.T) {
516516
tokensSaved, _ := meta["tokens_saved"].(float64)
517517
baselineTokens, _ := meta["baseline_tokens"].(float64)
518518
responseTokens, _ := meta["response_tokens"].(float64)
519-
compressionRatio, _ := meta["compression_ratio"].(float64)
519+
reductionRatio, _ := meta["reduction_ratio"].(float64)
520520

521521
if tokensSaved < 0 {
522522
t.Errorf("tokens_saved should be >= 0, got %v", tokensSaved)
@@ -527,7 +527,7 @@ func TestGetCodeSnippet_MetaField(t *testing.T) {
527527
if responseTokens <= 0 {
528528
t.Errorf("response_tokens should be > 0, got %v", responseTokens)
529529
}
530-
if compressionRatio <= 0 {
531-
t.Errorf("compression_ratio should be > 0, got %v", compressionRatio)
530+
if reductionRatio <= 0 {
531+
t.Errorf("reduction_ratio should be > 0, got %v", reductionRatio)
532532
}
533533
}

‎internal/tools/tools.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -814,6 +814,14 @@ func (s *Server) initMetricsTracker() {
814814

815815
// resultWithMeta wraps data with a _meta field containing token savings estimation.
816816
// If tracker is non-nil, also records the savings cumulatively.
817+
//
818+
// Currently instrumented: search_graph, get_code_snippet.
819+
// NOT instrumented (intentionally): trace_call_path, query_graph, search_code,
820+
// get_architecture, get_graph_schema, detect_changes, index_repository,
821+
// list_projects, index_status, manage_adr, delete_project.
822+
// Rationale: validating the _meta pattern on high-value tools first; tools like
823+
// index_repository and list_projects have no file-read equivalent so savings = 0.
824+
// Use this wrapper to add more tools incrementally.
817825
func resultWithMeta(data map[string]any, meta metrics.TokenMetadata, tracker *metrics.Tracker) *mcp.CallToolResult {
818826
data["_meta"] = meta
819827
if tracker != nil {

‎internal/tools/tools_test.go‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ package tools
33
import (
44
"runtime"
55
"testing"
6+
7+
"github.com/DeusData/codebase-memory-mcp/internal/store"
68
)
79

810
func TestParseFileURI(t *testing.T) {
@@ -51,6 +53,93 @@ func TestParseFileURI(t *testing.T) {
5153
}
5254
}
5355

56+
func TestPriceForConfig(t *testing.T) {
57+
tests := []struct {
58+
name string
59+
cfg func(t *testing.T) *store.ConfigStore
60+
want float64
61+
}{
62+
{"nil config", func(t *testing.T) *store.ConfigStore { return nil }, 0.000015},
63+
{"default (claude-sonnet)", func(t *testing.T) *store.ConfigStore {
64+
c, err := store.OpenConfigInDir(t.TempDir())
65+
if err != nil {
66+
t.Fatal(err)
67+
}
68+
t.Cleanup(func() { c.Close() })
69+
return c
70+
}, 0.000015},
71+
{"claude-opus", func(t *testing.T) *store.ConfigStore {
72+
c, err := store.OpenConfigInDir(t.TempDir())
73+
if err != nil {
74+
t.Fatal(err)
75+
}
76+
t.Cleanup(func() { c.Close() })
77+
c.Set(store.ConfigPricingModel, "claude-opus")
78+
return c
79+
}, 0.000075},
80+
{"gpt-4o", func(t *testing.T) *store.ConfigStore {
81+
c, err := store.OpenConfigInDir(t.TempDir())
82+
if err != nil {
83+
t.Fatal(err)
84+
}
85+
t.Cleanup(func() { c.Close() })
86+
c.Set(store.ConfigPricingModel, "gpt-4o")
87+
return c
88+
}, 0.000010},
89+
{"custom", func(t *testing.T) *store.ConfigStore {
90+
c, err := store.OpenConfigInDir(t.TempDir())
91+
if err != nil {
92+
t.Fatal(err)
93+
}
94+
t.Cleanup(func() { c.Close() })
95+
c.Set(store.ConfigPricingModel, "custom")
96+
c.Set(store.ConfigCustomPricePerToken, "0.00005")
97+
return c
98+
}, 0.00005},
99+
}
100+
for _, tt := range tests {
101+
t.Run(tt.name, func(t *testing.T) {
102+
got := priceForConfig(tt.cfg(t))
103+
if got != tt.want {
104+
t.Errorf("priceForConfig() = %v, want %v", got, tt.want)
105+
}
106+
})
107+
}
108+
}
109+
110+
func TestInitMetricsTracker(t *testing.T) {
111+
t.Run("default enabled", func(t *testing.T) {
112+
router, err := store.NewRouterWithDir(t.TempDir())
113+
if err != nil {
114+
t.Fatal(err)
115+
}
116+
t.Cleanup(router.CloseAll)
117+
srv := NewServer(router)
118+
if srv.metricsTracker == nil {
119+
t.Error("expected metricsTracker to be non-nil by default")
120+
}
121+
})
122+
123+
t.Run("disabled via config", func(t *testing.T) {
124+
router, err := store.NewRouterWithDir(t.TempDir())
125+
if err != nil {
126+
t.Fatal(err)
127+
}
128+
t.Cleanup(router.CloseAll)
129+
cfg, err := store.OpenConfigInDir(t.TempDir())
130+
if err != nil {
131+
t.Fatal(err)
132+
}
133+
t.Cleanup(func() { cfg.Close() })
134+
cfg.Set(store.ConfigMetricsEnabled, "false")
135+
136+
srv := NewServer(router, WithConfig(cfg))
137+
if srv.metricsTracker != nil {
138+
t.Error("expected metricsTracker to be nil when disabled")
139+
}
140+
})
141+
}
142+
54143
// windowsPath converts forward slashes to backslashes for Windows comparison.
55144
func windowsPath(p string) string {
56145
result := make([]byte, len(p))

0 commit comments

Comments
 (0)