Skip to content

Commit c5d070e

Browse files
committed
fix: implement chronological deduplication for branched conversations
- Add deduplication logic that respects chronological order to fix issue #19 - Sort JSONL files by earliest timestamp before processing - Track processed message+request ID combinations to skip duplicates - Ensure original entries are kept and later duplicates are discarded - Add comprehensive test suite for deduplication functionality This prevents double-counting of tokens and costs when conversations are branched, which creates duplicate entries with identical message and request IDs. Files are now processed oldest-first to ensure costs are attributed to the correct date.
1 parent 55a3f6a commit c5d070e

2 files changed

Lines changed: 459 additions & 2 deletions

File tree

Lines changed: 334 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,334 @@
1+
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
2+
import { tmpdir } from 'node:os';
3+
import path from 'node:path';
4+
import { describe, expect, it } from 'bun:test';
5+
import {
6+
createUniqueHash,
7+
getEarliestTimestamp,
8+
loadDailyUsageData,
9+
loadSessionData,
10+
sortFilesByTimestamp,
11+
} from './data-loader.ts';
12+
13+
describe('deduplication functionality', () => {
14+
describe('createUniqueHash', () => {
15+
it('should create hash from message id and request id', () => {
16+
const data = {
17+
timestamp: '2025-01-10T10:00:00Z',
18+
message: {
19+
id: 'msg_123',
20+
usage: {
21+
input_tokens: 100,
22+
output_tokens: 50,
23+
},
24+
},
25+
requestId: 'req_456',
26+
};
27+
28+
const hash = createUniqueHash(data);
29+
expect(hash).toBe('msg_123:req_456');
30+
});
31+
32+
it('should return null when message id is missing', () => {
33+
const data = {
34+
timestamp: '2025-01-10T10:00:00Z',
35+
message: {
36+
usage: {
37+
input_tokens: 100,
38+
output_tokens: 50,
39+
},
40+
},
41+
requestId: 'req_456',
42+
};
43+
44+
const hash = createUniqueHash(data);
45+
expect(hash).toBeNull();
46+
});
47+
48+
it('should return null when request id is missing', () => {
49+
const data = {
50+
timestamp: '2025-01-10T10:00:00Z',
51+
message: {
52+
id: 'msg_123',
53+
usage: {
54+
input_tokens: 100,
55+
output_tokens: 50,
56+
},
57+
},
58+
};
59+
60+
const hash = createUniqueHash(data);
61+
expect(hash).toBeNull();
62+
});
63+
});
64+
65+
describe('getEarliestTimestamp', () => {
66+
let tempDir: string;
67+
68+
it('should extract earliest timestamp from JSONL file', async () => {
69+
tempDir = await mkdtemp(path.join(tmpdir(), 'ccusage-test-'));
70+
const testFile = path.join(tempDir, 'test.jsonl');
71+
72+
const content = [
73+
JSON.stringify({ timestamp: '2025-01-15T12:00:00Z', message: { usage: {} } }),
74+
JSON.stringify({ timestamp: '2025-01-10T10:00:00Z', message: { usage: {} } }),
75+
JSON.stringify({ timestamp: '2025-01-12T11:00:00Z', message: { usage: {} } }),
76+
].join('\n');
77+
78+
await writeFile(testFile, content);
79+
80+
const timestamp = await getEarliestTimestamp(testFile);
81+
expect(timestamp).toEqual(new Date('2025-01-10T10:00:00Z'));
82+
83+
await rm(tempDir, { recursive: true });
84+
});
85+
86+
it('should handle files without timestamps', async () => {
87+
tempDir = await mkdtemp(path.join(tmpdir(), 'ccusage-test-'));
88+
const testFile = path.join(tempDir, 'test.jsonl');
89+
90+
const content = [
91+
JSON.stringify({ message: { usage: {} } }),
92+
JSON.stringify({ data: 'no timestamp' }),
93+
].join('\n');
94+
95+
await writeFile(testFile, content);
96+
97+
const timestamp = await getEarliestTimestamp(testFile);
98+
expect(timestamp).toBeNull();
99+
100+
await rm(tempDir, { recursive: true });
101+
});
102+
103+
it('should skip invalid JSON lines', async () => {
104+
tempDir = await mkdtemp(path.join(tmpdir(), 'ccusage-test-'));
105+
const testFile = path.join(tempDir, 'test.jsonl');
106+
107+
const content = [
108+
'invalid json',
109+
JSON.stringify({ timestamp: '2025-01-10T10:00:00Z', message: { usage: {} } }),
110+
'{ broken: json',
111+
].join('\n');
112+
113+
await writeFile(testFile, content);
114+
115+
const timestamp = await getEarliestTimestamp(testFile);
116+
expect(timestamp).toEqual(new Date('2025-01-10T10:00:00Z'));
117+
118+
await rm(tempDir, { recursive: true });
119+
});
120+
});
121+
122+
describe('sortFilesByTimestamp', () => {
123+
let tempDir: string;
124+
125+
it('should sort files by earliest timestamp', async () => {
126+
tempDir = await mkdtemp(path.join(tmpdir(), 'ccusage-test-'));
127+
128+
const file1 = path.join(tempDir, 'file1.jsonl');
129+
const file2 = path.join(tempDir, 'file2.jsonl');
130+
const file3 = path.join(tempDir, 'file3.jsonl');
131+
132+
// File 1: earliest timestamp 2025-01-15
133+
await writeFile(file1, JSON.stringify({ timestamp: '2025-01-15T10:00:00Z' }));
134+
135+
// File 2: earliest timestamp 2025-01-10
136+
await writeFile(file2, JSON.stringify({ timestamp: '2025-01-10T10:00:00Z' }));
137+
138+
// File 3: earliest timestamp 2025-01-12
139+
await writeFile(file3, JSON.stringify({ timestamp: '2025-01-12T10:00:00Z' }));
140+
141+
const sorted = await sortFilesByTimestamp([file1, file2, file3]);
142+
143+
expect(sorted).toEqual([file2, file3, file1]); // Chronological order
144+
145+
await rm(tempDir, { recursive: true });
146+
});
147+
148+
it('should place files without timestamps at the end', async () => {
149+
tempDir = await mkdtemp(path.join(tmpdir(), 'ccusage-test-'));
150+
151+
const file1 = path.join(tempDir, 'file1.jsonl');
152+
const file2 = path.join(tempDir, 'file2.jsonl');
153+
const file3 = path.join(tempDir, 'file3.jsonl');
154+
155+
await writeFile(file1, JSON.stringify({ timestamp: '2025-01-15T10:00:00Z' }));
156+
await writeFile(file2, JSON.stringify({ no_timestamp: true }));
157+
await writeFile(file3, JSON.stringify({ timestamp: '2025-01-10T10:00:00Z' }));
158+
159+
const sorted = await sortFilesByTimestamp([file1, file2, file3]);
160+
161+
expect(sorted).toEqual([file3, file1, file2]); // file2 without timestamp goes to end
162+
163+
await rm(tempDir, { recursive: true });
164+
});
165+
});
166+
167+
describe('loadDailyUsageData with deduplication', () => {
168+
let tempDir: string;
169+
170+
it('should deduplicate entries with same message and request IDs', async () => {
171+
tempDir = await mkdtemp(path.join(tmpdir(), 'ccusage-test-'));
172+
const projectDir = path.join(tempDir, 'projects');
173+
const session1Dir = path.join(projectDir, 'project1', 'session1');
174+
const session2Dir = path.join(projectDir, 'project1', 'session2');
175+
176+
await mkdir(session1Dir, { recursive: true });
177+
await writeFile(path.join(session1Dir, 'file1.jsonl'), JSON.stringify({
178+
timestamp: '2025-01-10T10:00:00Z',
179+
message: {
180+
id: 'msg_123',
181+
usage: {
182+
input_tokens: 100,
183+
output_tokens: 50,
184+
},
185+
},
186+
requestId: 'req_456',
187+
costUSD: 0.001,
188+
}));
189+
190+
// Duplicate entry in a later session
191+
await mkdir(session2Dir, { recursive: true });
192+
await writeFile(path.join(session2Dir, 'file2.jsonl'), JSON.stringify({
193+
timestamp: '2025-01-15T10:00:00Z',
194+
message: {
195+
id: 'msg_123',
196+
usage: {
197+
input_tokens: 100,
198+
output_tokens: 50,
199+
},
200+
},
201+
requestId: 'req_456',
202+
costUSD: 0.001,
203+
}));
204+
205+
const data = await loadDailyUsageData({
206+
claudePath: tempDir,
207+
mode: 'display',
208+
});
209+
210+
// Should only have one entry for 2025-01-10
211+
expect(data).toHaveLength(1);
212+
expect(data[0]?.date).toBe('2025-01-10');
213+
expect(data[0]?.inputTokens).toBe(100);
214+
expect(data[0]?.outputTokens).toBe(50);
215+
216+
await rm(tempDir, { recursive: true });
217+
});
218+
219+
it('should process files in chronological order', async () => {
220+
tempDir = await mkdtemp(path.join(tmpdir(), 'ccusage-test-'));
221+
const projectDir = path.join(tempDir, 'projects');
222+
223+
// Create files with different timestamps but same message/request IDs
224+
const newerFile = path.join(projectDir, 'newer.jsonl');
225+
const olderFile = path.join(projectDir, 'older.jsonl');
226+
227+
// Newer file has an earlier entry in the file
228+
await mkdir(projectDir, { recursive: true });
229+
await writeFile(newerFile, JSON.stringify({
230+
timestamp: '2025-01-15T10:00:00Z',
231+
message: {
232+
id: 'msg_123',
233+
usage: {
234+
input_tokens: 200,
235+
output_tokens: 100,
236+
},
237+
},
238+
requestId: 'req_456',
239+
costUSD: 0.002,
240+
}));
241+
242+
// Older file has the original entry
243+
await writeFile(olderFile, JSON.stringify({
244+
timestamp: '2025-01-10T10:00:00Z',
245+
message: {
246+
id: 'msg_123',
247+
usage: {
248+
input_tokens: 100,
249+
output_tokens: 50,
250+
},
251+
},
252+
requestId: 'req_456',
253+
costUSD: 0.001,
254+
}));
255+
256+
const data = await loadDailyUsageData({
257+
claudePath: tempDir,
258+
mode: 'display',
259+
});
260+
261+
// Should keep the older entry (100/50 tokens) not the newer one (200/100)
262+
expect(data).toHaveLength(1);
263+
expect(data[0]?.date).toBe('2025-01-10');
264+
expect(data[0]?.inputTokens).toBe(100);
265+
expect(data[0]?.outputTokens).toBe(50);
266+
267+
await rm(tempDir, { recursive: true });
268+
});
269+
});
270+
271+
describe('loadSessionData with deduplication', () => {
272+
let tempDir: string;
273+
274+
it('should deduplicate entries across sessions', async () => {
275+
tempDir = await mkdtemp(path.join(tmpdir(), 'ccusage-test-'));
276+
const projectDir = path.join(tempDir, 'projects');
277+
const session1Dir = path.join(projectDir, 'project1', 'session1');
278+
const session2Dir = path.join(projectDir, 'project1', 'session2');
279+
280+
await mkdir(session1Dir, { recursive: true });
281+
await writeFile(path.join(session1Dir, 'file1.jsonl'), JSON.stringify({
282+
timestamp: '2025-01-10T10:00:00Z',
283+
message: {
284+
id: 'msg_123',
285+
usage: {
286+
input_tokens: 100,
287+
output_tokens: 50,
288+
},
289+
},
290+
requestId: 'req_456',
291+
costUSD: 0.001,
292+
}));
293+
294+
// Duplicate entry in session2
295+
await mkdir(session2Dir, { recursive: true });
296+
await writeFile(path.join(session2Dir, 'file2.jsonl'), JSON.stringify({
297+
timestamp: '2025-01-15T10:00:00Z',
298+
message: {
299+
id: 'msg_123',
300+
usage: {
301+
input_tokens: 100,
302+
output_tokens: 50,
303+
},
304+
},
305+
requestId: 'req_456',
306+
costUSD: 0.001,
307+
}));
308+
309+
const sessions = await loadSessionData({
310+
claudePath: tempDir,
311+
mode: 'display',
312+
});
313+
314+
// Session 1 should have the entry
315+
const session1 = sessions.find(s => s.sessionId === 'session1');
316+
expect(session1).toBeDefined();
317+
expect(session1?.inputTokens).toBe(100);
318+
expect(session1?.outputTokens).toBe(50);
319+
320+
// Session 2 should either not exist or have 0 tokens (duplicate was skipped)
321+
const session2 = sessions.find(s => s.sessionId === 'session2');
322+
if (session2 != null) {
323+
expect(session2.inputTokens).toBe(0);
324+
expect(session2.outputTokens).toBe(0);
325+
}
326+
else {
327+
// It's also valid for session2 to not be included if it has no entries
328+
expect(sessions.length).toBe(1);
329+
}
330+
331+
await rm(tempDir, { recursive: true });
332+
});
333+
});
334+
});

0 commit comments

Comments
 (0)