Skip to content

Commit d6f7666

Browse files
committed
fix(telemetry): send reason, count and limit on consecutive mistake event
1 parent 0ac21e8 commit d6f7666

6 files changed

Lines changed: 81 additions & 10 deletions

File tree

‎packages/telemetry/src/TelemetryService.ts‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { ZodError } from "zod"
22

33
import {
4+
type ConsecutiveMistakeReason,
45
type TelemetryClient,
56
type TelemetryPropertiesProvider,
67
TelemetryEventName,
@@ -280,8 +281,18 @@ export class TelemetryService {
280281
this.captureEvent(TelemetryEventName.SHELL_INTEGRATION_ERROR, { taskId })
281282
}
282283

283-
public captureConsecutiveMistakeError(taskId: string): void {
284-
this.captureEvent(TelemetryEventName.CONSECUTIVE_MISTAKE_ERROR, { taskId })
284+
public captureConsecutiveMistakeError(
285+
taskId: string,
286+
reason: ConsecutiveMistakeReason,
287+
consecutiveMistakeCount: number,
288+
consecutiveMistakeLimit: number,
289+
): void {
290+
this.captureEvent(TelemetryEventName.CONSECUTIVE_MISTAKE_ERROR, {
291+
taskId,
292+
reason,
293+
consecutiveMistakeCount,
294+
consecutiveMistakeLimit,
295+
})
285296
}
286297

287298
/**
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// pnpm --filter @roo-code/telemetry test src/__tests__/TelemetryService.consecutive-mistake.test.ts
2+
3+
import { TelemetryEventName, type TelemetryClient } from "@roo-code/types"
4+
5+
import { TelemetryService } from "../TelemetryService"
6+
7+
describe("TelemetryService.captureConsecutiveMistakeError", () => {
8+
let mockClient: TelemetryClient
9+
10+
beforeEach(() => {
11+
mockClient = {
12+
setProvider: vi.fn(),
13+
capture: vi.fn().mockResolvedValue(undefined),
14+
captureException: vi.fn().mockResolvedValue(undefined),
15+
updateTelemetryState: vi.fn(),
16+
isTelemetryEnabled: vi.fn().mockReturnValue(true),
17+
shutdown: vi.fn().mockResolvedValue(undefined),
18+
}
19+
})
20+
21+
it.each(["no_tools_used", "tool_repetition"] as const)(
22+
"sends reason, count and limit for %s without a separate exception",
23+
(reason) => {
24+
const service = new TelemetryService([mockClient])
25+
26+
service.captureConsecutiveMistakeError("task_1", reason, 3, 5)
27+
28+
expect(mockClient.capture).toHaveBeenCalledWith({
29+
event: TelemetryEventName.CONSECUTIVE_MISTAKE_ERROR,
30+
properties: {
31+
taskId: "task_1",
32+
reason,
33+
consecutiveMistakeCount: 3,
34+
consecutiveMistakeLimit: 5,
35+
},
36+
})
37+
expect(mockClient.captureException).not.toHaveBeenCalled()
38+
},
39+
)
40+
})

‎src/api/providers/__tests__/openai-native.spec.ts‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1566,10 +1566,9 @@ describe("OpenAiNativeHandler", () => {
15661566
}
15671567
}).rejects.toThrow()
15681568

1569-
// Verify telemetry was captured (may be called multiple times due to error propagation)
1570-
expect(mockCaptureException).toHaveBeenCalled()
1569+
// A stream failure must be captured exactly once (handleStreamResponse rethrows to createMessage).
1570+
expect(mockCaptureException).toHaveBeenCalledTimes(1)
15711571

1572-
// Find the call with the stream error message
15731572
const streamErrorCall = mockCaptureException.mock.calls.find((call: any[]) =>
15741573
call[0]?.message?.includes("Model overloaded"),
15751574
)

‎src/core/assistant-message/__tests__/presentAssistantMessage-tool-repetition.spec.ts‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -297,7 +297,12 @@ describe("presentAssistantMessage - Tool Repetition Detection", () => {
297297
expect.stringContaining("tools:toolRepetitionLimitReached"),
298298
)
299299
expect(readFileHandle).not.toHaveBeenCalled()
300-
expect(captureConsecutiveMistakeError).toHaveBeenCalledWith("test-task-id")
300+
expect(captureConsecutiveMistakeError).toHaveBeenCalledWith(
301+
"test-task-id",
302+
"tool_repetition",
303+
mockTask.consecutiveMistakeCount,
304+
HARD_LIMIT,
305+
)
301306
expect(findToolResult(nextToolCallId)).toBeDefined()
302307
})
303308

@@ -361,8 +366,14 @@ describe("presentAssistantMessage - Tool Repetition Detection", () => {
361366
expect(readFileHandle).not.toHaveBeenCalled()
362367

363368
// Telemetry escalation should have fired for a hard block.
364-
expect(captureConsecutiveMistakeError).toHaveBeenCalledWith("test-task-id")
365-
expect(captureException).toHaveBeenCalled()
369+
expect(captureConsecutiveMistakeError).toHaveBeenCalledWith(
370+
"test-task-id",
371+
"tool_repetition",
372+
mockTask.consecutiveMistakeCount,
373+
HARD_LIMIT,
374+
)
375+
// The named event is the single capture: no duplicate exception.
376+
expect(captureException).not.toHaveBeenCalled()
366377

367378
// A tool_result describing the repetition limit should have been pushed.
368379
const toolResult = findToolResult(toolCallId)

‎src/core/assistant-message/presentAssistantMessage.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -836,7 +836,12 @@ async function presentAssistantMessageBlock(cline: Task): Promise<void> {
836836
}
837837

838838
// Track tool repetition in telemetry via a named event.
839-
TelemetryService.instance.captureConsecutiveMistakeError(cline.taskId)
839+
TelemetryService.instance.captureConsecutiveMistakeError(
840+
cline.taskId,
841+
"tool_repetition",
842+
cline.consecutiveMistakeCount,
843+
cline.consecutiveMistakeLimit,
844+
)
840845

841846
// Return tool result message about the repetition
842847
pushToolResult(

‎src/core/task/Task.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3784,7 +3784,12 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
37843784
// Track consecutive mistake errors in telemetry via a named event.
37853785
// The reason is "no_tools_used" because this limit is reached via initiateTaskLoop
37863786
// which increments consecutiveMistakeCount when the model doesn't use any tools.
3787-
TelemetryService.instance.captureConsecutiveMistakeError(this.taskId)
3787+
TelemetryService.instance.captureConsecutiveMistakeError(
3788+
this.taskId,
3789+
"no_tools_used",
3790+
this.consecutiveMistakeCount,
3791+
this.consecutiveMistakeLimit,
3792+
)
37883793

37893794
const { response, text, images } = await this.ask(
37903795
"mistake_limit_reached",

0 commit comments

Comments
 (0)