Skip to content

Commit 36fd76d

Browse files
committed
fix: resolve duplicate message id
✅ Core Implementation 1. Extended Schema: Added optional requestId and message.id fields to the UsageDataSchema 2. Deduplication Logic: Implemented createUniqueHash() function that creates unique identifiers from message ID + request ID combinations 3. Global Deduplication: Added deduplication across all files in both loadDailyUsageData() and loadSessionData() functions 4. Backward Compatibility: Entries without IDs are processed normally (no breaking changes) ✅ Key Features - Prevents Double-Counting: Skips duplicate message+request ID combinations globally across all conversation branches - Handles Edge Cases: Processes entries without IDs normally, allows same message ID with different request IDs - Session Integrity: Ensures session entries are created even if all messages are deduplicated (prevents missing sessions) - Cost Accuracy: Only calculates costs for unique messages, preventing inflated usage numbers in branched conversations
1 parent 5fbaede commit 36fd76d

2 files changed

Lines changed: 344 additions & 0 deletions

File tree

‎src/data-loader.test.ts‎

Lines changed: 299 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { createFixture } from "fs-fixture";
44
import {
55
type UsageData,
66
calculateCostForEntry,
7+
createUniqueHash,
78
formatDate,
89
loadDailyUsageData,
910
loadMonthlyUsageData,
@@ -34,6 +35,304 @@ describe("formatDate", () => {
3435
});
3536
});
3637

38+
describe("createUniqueHash", () => {
39+
test("creates hash when both message ID and request ID are present", () => {
40+
const data: UsageData = {
41+
timestamp: "2024-01-01T00:00:00Z",
42+
message: {
43+
usage: { input_tokens: 100, output_tokens: 50 },
44+
id: "msg123",
45+
},
46+
requestId: "req456",
47+
};
48+
49+
const hash = createUniqueHash(data);
50+
expect(hash).toBe("msg123:req456");
51+
});
52+
53+
test("returns null when message ID is missing", () => {
54+
const data: UsageData = {
55+
timestamp: "2024-01-01T00:00:00Z",
56+
message: {
57+
usage: { input_tokens: 100, output_tokens: 50 },
58+
},
59+
requestId: "req456",
60+
};
61+
62+
const hash = createUniqueHash(data);
63+
expect(hash).toBeNull();
64+
});
65+
66+
test("returns null when request ID is missing", () => {
67+
const data: UsageData = {
68+
timestamp: "2024-01-01T00:00:00Z",
69+
message: {
70+
usage: { input_tokens: 100, output_tokens: 50 },
71+
id: "msg123",
72+
},
73+
};
74+
75+
const hash = createUniqueHash(data);
76+
expect(hash).toBeNull();
77+
});
78+
79+
test("returns null when both IDs are missing", () => {
80+
const data: UsageData = {
81+
timestamp: "2024-01-01T00:00:00Z",
82+
message: {
83+
usage: { input_tokens: 100, output_tokens: 50 },
84+
},
85+
};
86+
87+
const hash = createUniqueHash(data);
88+
expect(hash).toBeNull();
89+
});
90+
});
91+
92+
describe("deduplication functionality", () => {
93+
describe("loadDailyUsageData with deduplication", () => {
94+
test("should skip duplicate message+request ID combinations", async () => {
95+
const originalData = {
96+
timestamp: "2024-01-01T00:00:00Z",
97+
message: {
98+
usage: { input_tokens: 100, output_tokens: 50 },
99+
id: "msg123",
100+
},
101+
requestId: "req456",
102+
costUSD: 0.01,
103+
};
104+
105+
const duplicateData = {
106+
timestamp: "2024-01-01T01:00:00Z", // Different timestamp
107+
message: {
108+
usage: { input_tokens: 200, output_tokens: 100 }, // Different token counts
109+
id: "msg123", // Same message ID
110+
},
111+
requestId: "req456", // Same request ID
112+
costUSD: 0.02, // Different cost
113+
};
114+
115+
const differentData = {
116+
timestamp: "2024-01-01T02:00:00Z",
117+
message: {
118+
usage: { input_tokens: 300, output_tokens: 150 },
119+
id: "msg789", // Different message ID
120+
},
121+
requestId: "req456", // Same request ID but different message ID
122+
costUSD: 0.03,
123+
};
124+
125+
await using fixture = await createFixture({
126+
projects: {
127+
project1: {
128+
session1: {
129+
"original.jsonl": JSON.stringify(originalData),
130+
},
131+
session2: {
132+
"duplicate.jsonl": JSON.stringify(duplicateData),
133+
"different.jsonl": JSON.stringify(differentData),
134+
},
135+
},
136+
},
137+
});
138+
139+
const result = await loadDailyUsageData({ claudePath: fixture.path });
140+
141+
expect(result).toHaveLength(1);
142+
expect(result[0]?.date).toBe("2024-01-01");
143+
// Should only include original and different data, not duplicate
144+
expect(result[0]?.inputTokens).toBe(400); // 100 + 300 (not 200 from duplicate)
145+
expect(result[0]?.outputTokens).toBe(200); // 50 + 150 (not 100 from duplicate)
146+
expect(result[0]?.totalCost).toBe(0.04); // 0.01 + 0.03 (not 0.02 from duplicate)
147+
});
148+
149+
test("should process entries without message ID and request ID normally", async () => {
150+
const dataWithIds = {
151+
timestamp: "2024-01-01T00:00:00Z",
152+
message: {
153+
usage: { input_tokens: 100, output_tokens: 50 },
154+
id: "msg123",
155+
},
156+
requestId: "req456",
157+
costUSD: 0.01,
158+
};
159+
160+
const dataWithoutIds = {
161+
timestamp: "2024-01-01T01:00:00Z",
162+
message: {
163+
usage: { input_tokens: 200, output_tokens: 100 },
164+
},
165+
costUSD: 0.02,
166+
};
167+
168+
await using fixture = await createFixture({
169+
projects: {
170+
project1: {
171+
session1: {
172+
"usage.jsonl": `${JSON.stringify(dataWithIds)}\n${JSON.stringify(dataWithoutIds)}`,
173+
},
174+
},
175+
},
176+
});
177+
178+
const result = await loadDailyUsageData({ claudePath: fixture.path });
179+
180+
expect(result).toHaveLength(1);
181+
expect(result[0]?.inputTokens).toBe(300); // Both entries should be processed
182+
expect(result[0]?.outputTokens).toBe(150);
183+
expect(result[0]?.totalCost).toBe(0.03);
184+
});
185+
186+
test("should allow same message ID with different request IDs", async () => {
187+
const data1 = {
188+
timestamp: "2024-01-01T00:00:00Z",
189+
message: {
190+
usage: { input_tokens: 100, output_tokens: 50 },
191+
id: "msg123", // Same message ID
192+
},
193+
requestId: "req456", // Different request ID
194+
costUSD: 0.01,
195+
};
196+
197+
const data2 = {
198+
timestamp: "2024-01-01T01:00:00Z",
199+
message: {
200+
usage: { input_tokens: 200, output_tokens: 100 },
201+
id: "msg123", // Same message ID
202+
},
203+
requestId: "req789", // Different request ID
204+
costUSD: 0.02,
205+
};
206+
207+
await using fixture = await createFixture({
208+
projects: {
209+
project1: {
210+
session1: {
211+
"usage.jsonl": `${JSON.stringify(data1)}\n${JSON.stringify(data2)}`,
212+
},
213+
},
214+
},
215+
});
216+
217+
const result = await loadDailyUsageData({ claudePath: fixture.path });
218+
219+
expect(result).toHaveLength(1);
220+
expect(result[0]?.inputTokens).toBe(300); // Both should be processed (different combinations)
221+
expect(result[0]?.outputTokens).toBe(150);
222+
expect(result[0]?.totalCost).toBe(0.03);
223+
});
224+
});
225+
226+
describe("loadSessionData with deduplication", () => {
227+
test("should skip duplicate message+request ID combinations across sessions", async () => {
228+
const originalData = {
229+
timestamp: "2024-01-01T00:00:00Z",
230+
message: {
231+
usage: { input_tokens: 100, output_tokens: 50 },
232+
id: "msg123",
233+
},
234+
requestId: "req456",
235+
costUSD: 0.01,
236+
};
237+
238+
const duplicateData = {
239+
timestamp: "2024-01-01T01:00:00Z",
240+
message: {
241+
usage: { input_tokens: 200, output_tokens: 100 },
242+
id: "msg123", // Same message ID
243+
},
244+
requestId: "req456", // Same request ID
245+
costUSD: 0.02,
246+
};
247+
248+
await using fixture = await createFixture({
249+
projects: {
250+
project1: {
251+
session1: {
252+
"original.jsonl": JSON.stringify(originalData),
253+
},
254+
session2: {
255+
"duplicate.jsonl": JSON.stringify(duplicateData),
256+
},
257+
},
258+
},
259+
});
260+
261+
const result = await loadSessionData({ claudePath: fixture.path });
262+
263+
expect(result).toHaveLength(2); // Two sessions
264+
265+
// Find the sessions
266+
const session1 = result.find((s) => s.sessionId === "session1");
267+
const session2 = result.find((s) => s.sessionId === "session2");
268+
269+
expect(session1).toBeTruthy();
270+
expect(session2).toBeTruthy();
271+
272+
// Only original data should be counted
273+
expect(session1?.inputTokens).toBe(100);
274+
expect(session1?.outputTokens).toBe(50);
275+
expect(session1?.totalCost).toBe(0.01);
276+
277+
// Duplicate should be skipped, so session2 should have zero usage
278+
expect(session2?.inputTokens).toBe(0);
279+
expect(session2?.outputTokens).toBe(0);
280+
expect(session2?.totalCost).toBe(0);
281+
});
282+
283+
test("should properly handle deduplication within same session", async () => {
284+
const originalData = {
285+
timestamp: "2024-01-01T00:00:00Z",
286+
message: {
287+
usage: { input_tokens: 100, output_tokens: 50 },
288+
id: "msg123",
289+
},
290+
requestId: "req456",
291+
costUSD: 0.01,
292+
};
293+
294+
const duplicateData = {
295+
timestamp: "2024-01-01T01:00:00Z",
296+
message: {
297+
usage: { input_tokens: 200, output_tokens: 100 },
298+
id: "msg123", // Same message ID
299+
},
300+
requestId: "req456", // Same request ID
301+
costUSD: 0.02,
302+
};
303+
304+
const uniqueData = {
305+
timestamp: "2024-01-01T02:00:00Z",
306+
message: {
307+
usage: { input_tokens: 300, output_tokens: 150 },
308+
id: "msg789", // Different message ID
309+
},
310+
requestId: "req456", // Same request ID but different message
311+
costUSD: 0.03,
312+
};
313+
314+
await using fixture = await createFixture({
315+
projects: {
316+
project1: {
317+
session1: {
318+
"usage.jsonl": `${JSON.stringify(originalData)}\n${JSON.stringify(duplicateData)}\n${JSON.stringify(uniqueData)}`,
319+
},
320+
},
321+
},
322+
});
323+
324+
const result = await loadSessionData({ claudePath: fixture.path });
325+
326+
expect(result).toHaveLength(1);
327+
expect(result[0]?.sessionId).toBe("session1");
328+
// Should only include original and unique data, not duplicate
329+
expect(result[0]?.inputTokens).toBe(400); // 100 + 300 (not 200 from duplicate)
330+
expect(result[0]?.outputTokens).toBe(200); // 50 + 150 (not 100 from duplicate)
331+
expect(result[0]?.totalCost).toBe(0.04); // 0.01 + 0.03 (not 0.02 from duplicate)
332+
});
333+
});
334+
});
335+
37336
describe("loadDailyUsageData", () => {
38337
test("returns empty array when no files found", async () => {
39338
await using fixture = await createFixture({

0 commit comments

Comments
 (0)