Skip to content

Commit adee4d2

Browse files
myk1ytZoo (VP)edelauna
authored
fix(telemetry): remove redundant captureException calls that double-count errors (Zoo-Code-Org#1702)
* fix(telemetry): remove redundant captureException calls that double-count errors Issue Zoo-Code-Org#513 identified captureException sites where the same error is captured again downstream or where a named telemetry event already carries the signal: - Task.ts: captureException(new ConsecutiveMistakeError(...)) fired right after captureConsecutiveMistakeError(), routing a deliberate control-flow limit into error tracking as a duplicate. - presentAssistantMessage.ts: same redundant pairing for the tool_repetition guard. - openai-native.ts: handleStreamResponse captured stream-processing failures and rethrew, after which createMessage's catch captured the same failure again. The openrouter handleStreamingError capture was left in place: in current code its rethrown error is not captured anywhere downstream (createMessage has no enclosing catch for the stream loop and completePrompt invokes it outside its try/catch), so removing it would drop the only capture for mid-stream OpenRouter errors. * fix(telemetry): send reason, count and limit on consecutive mistake event * fix(telemetry): report accurate reason and count on consecutive mistake event --------- Co-authored-by: Zoo (VP) <[email protected]> Co-authored-by: Elliott de Launay <[email protected]>
1 parent fafad20 commit adee4d2

7 files changed

Lines changed: 89 additions & 40 deletions

File tree

‎packages/telemetry/src/TelemetryService.ts‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { ZodError } from "zod"
22

33
import {
4+
type ConsecutiveMistakeReason,
45
type TelemetryClient,
56
type TelemetryPropertiesProvider,
67
TelemetryEventName,
@@ -280,8 +281,18 @@ export class TelemetryService {
280281
this.captureEvent(TelemetryEventName.SHELL_INTEGRATION_ERROR, { taskId })
281282
}
282283

283-
public captureConsecutiveMistakeError(taskId: string): void {
284-
this.captureEvent(TelemetryEventName.CONSECUTIVE_MISTAKE_ERROR, { taskId })
284+
public captureConsecutiveMistakeError(
285+
taskId: string,
286+
reason: ConsecutiveMistakeReason,
287+
consecutiveMistakeCount: number,
288+
consecutiveMistakeLimit: number,
289+
): void {
290+
this.captureEvent(TelemetryEventName.CONSECUTIVE_MISTAKE_ERROR, {
291+
taskId,
292+
reason,
293+
consecutiveMistakeCount,
294+
consecutiveMistakeLimit,
295+
})
285296
}
286297

287298
/**
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// pnpm --filter @roo-code/telemetry test src/__tests__/TelemetryService.consecutive-mistake.test.ts
2+
3+
import { TelemetryEventName, type TelemetryClient } from "@roo-code/types"
4+
5+
import { TelemetryService } from "../TelemetryService"
6+
7+
describe("TelemetryService.captureConsecutiveMistakeError", () => {
8+
let mockClient: TelemetryClient
9+
10+
beforeEach(() => {
11+
mockClient = {
12+
setProvider: vi.fn(),
13+
capture: vi.fn().mockResolvedValue(undefined),
14+
captureException: vi.fn().mockResolvedValue(undefined),
15+
updateTelemetryState: vi.fn(),
16+
isTelemetryEnabled: vi.fn().mockReturnValue(true),
17+
shutdown: vi.fn().mockResolvedValue(undefined),
18+
}
19+
})
20+
21+
it.each(["no_tools_used", "tool_repetition"] as const)(
22+
"sends reason, count and limit for %s without a separate exception",
23+
(reason) => {
24+
const service = new TelemetryService([mockClient])
25+
26+
service.captureConsecutiveMistakeError("task_1", reason, 3, 5)
27+
28+
expect(mockClient.capture).toHaveBeenCalledWith({
29+
event: TelemetryEventName.CONSECUTIVE_MISTAKE_ERROR,
30+
properties: {
31+
taskId: "task_1",
32+
reason,
33+
consecutiveMistakeCount: 3,
34+
consecutiveMistakeLimit: 5,
35+
},
36+
})
37+
expect(mockClient.captureException).not.toHaveBeenCalled()
38+
},
39+
)
40+
})

‎src/api/providers/__tests__/openai-native.spec.ts‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1566,10 +1566,9 @@ describe("OpenAiNativeHandler", () => {
15661566
}
15671567
}).rejects.toThrow()
15681568

1569-
// Verify telemetry was captured (may be called multiple times due to error propagation)
1570-
expect(mockCaptureException).toHaveBeenCalled()
1569+
// A stream failure must be captured exactly once (handleStreamResponse rethrows to createMessage).
1570+
expect(mockCaptureException).toHaveBeenCalledTimes(1)
15711571

1572-
// Find the call with the stream error message
15731572
const streamErrorCall = mockCaptureException.mock.calls.find((call: any[]) =>
15741573
call[0]?.message?.includes("Model overloaded"),
15751574
)

‎src/api/providers/openai-native.ts‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1129,10 +1129,8 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
11291129
// If we didn't get any content, don't throw - the API might have returned an empty response
11301130
// This can happen in certain edge cases and shouldn't break the flow
11311131
} catch (error) {
1132-
const errorMessage = error instanceof Error ? error.message : String(error)
1133-
const apiError = new ApiProviderError(errorMessage, this.providerName, model.id, "createMessage")
1134-
TelemetryService.instance.captureException(apiError)
1135-
1132+
// No captureException here: the createMessage catch captures this
1133+
// failure once when the rethrown error propagates.
11361134
if (error instanceof Error) {
11371135
throw new Error(`Error processing response stream: ${error.message}`)
11381136
}

‎src/core/assistant-message/__tests__/presentAssistantMessage-tool-repetition.spec.ts‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -297,7 +297,12 @@ describe("presentAssistantMessage - Tool Repetition Detection", () => {
297297
expect.stringContaining("tools:toolRepetitionLimitReached"),
298298
)
299299
expect(readFileHandle).not.toHaveBeenCalled()
300-
expect(captureConsecutiveMistakeError).toHaveBeenCalledWith("test-task-id")
300+
expect(captureConsecutiveMistakeError).toHaveBeenCalledWith(
301+
"test-task-id",
302+
"tool_repetition",
303+
HARD_LIMIT,
304+
HARD_LIMIT,
305+
)
301306
expect(findToolResult(nextToolCallId)).toBeDefined()
302307
})
303308

@@ -361,8 +366,14 @@ describe("presentAssistantMessage - Tool Repetition Detection", () => {
361366
expect(readFileHandle).not.toHaveBeenCalled()
362367

363368
// Telemetry escalation should have fired for a hard block.
364-
expect(captureConsecutiveMistakeError).toHaveBeenCalledWith("test-task-id")
365-
expect(captureException).toHaveBeenCalled()
369+
expect(captureConsecutiveMistakeError).toHaveBeenCalledWith(
370+
"test-task-id",
371+
"tool_repetition",
372+
HARD_LIMIT,
373+
HARD_LIMIT,
374+
)
375+
// The named event is the single capture: no duplicate exception.
376+
expect(captureException).not.toHaveBeenCalled()
366377

367378
// A tool_result describing the repetition limit should have been pushed.
368379
const toolResult = findToolResult(toolCallId)

‎src/core/assistant-message/presentAssistantMessage.ts‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { serializeError } from "serialize-error"
22
import { Anthropic } from "@anthropic-ai/sdk"
33

44
import type { ToolName, ClineAsk, ToolProgressStatus } from "@roo-code/types"
5-
import { ConsecutiveMistakeError, TelemetryEventName } from "@roo-code/types"
5+
import { TelemetryEventName } from "@roo-code/types"
66
import { TelemetryService } from "@roo-code/telemetry"
77
import { customToolRegistry } from "@roo-code/core"
88

@@ -835,18 +835,15 @@ async function presentAssistantMessageBlock(cline: Task): Promise<void> {
835835
await cline.say("user_feedback", text, images)
836836
}
837837

838-
// Track tool repetition in telemetry via PostHog exception tracking and event.
839-
TelemetryService.instance.captureConsecutiveMistakeError(cline.taskId)
840-
TelemetryService.instance.captureException(
841-
new ConsecutiveMistakeError(
842-
`Tool repetition limit reached for ${block.name}`,
843-
cline.taskId,
844-
cline.consecutiveMistakeCount,
845-
cline.consecutiveMistakeLimit,
846-
"tool_repetition",
847-
cline.apiConfiguration.apiProvider,
848-
cline.api.getModel().id,
849-
),
838+
// Track tool repetition in telemetry via a named event.
839+
// The detector's hard limit is consecutiveMistakeLimit and it blocks when the
840+
// identical-call count reaches it, so that count equals the limit here.
841+
// consecutiveMistakeCount is unrelated and can be zero.
842+
TelemetryService.instance.captureConsecutiveMistakeError(
843+
cline.taskId,
844+
"tool_repetition",
845+
cline.consecutiveMistakeLimit,
846+
cline.consecutiveMistakeLimit,
850847
)
851848

852849
// Return tool result message about the repetition

‎src/core/task/Task.ts‎

Lines changed: 8 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,6 @@ import {
5555
DEFAULT_CHECKPOINT_TIMEOUT_SECONDS,
5656
MAX_CHECKPOINT_TIMEOUT_SECONDS,
5757
MIN_CHECKPOINT_TIMEOUT_SECONDS,
58-
ConsecutiveMistakeError,
5958
MAX_MCP_TOOLS_THRESHOLD,
6059
countEnabledMcpTools,
6160
providerIdentifiers,
@@ -3782,20 +3781,14 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
37823781
}
37833782

37843783
if (this.consecutiveMistakeLimit > 0 && this.consecutiveMistakeCount >= this.consecutiveMistakeLimit) {
3785-
// Track consecutive mistake errors in telemetry via event and PostHog exception tracking.
3786-
// The reason is "no_tools_used" because this limit is reached via initiateTaskLoop
3787-
// which increments consecutiveMistakeCount when the model doesn't use any tools.
3788-
TelemetryService.instance.captureConsecutiveMistakeError(this.taskId)
3789-
TelemetryService.instance.captureException(
3790-
new ConsecutiveMistakeError(
3791-
`Task reached consecutive mistake limit (${this.consecutiveMistakeLimit})`,
3792-
this.taskId,
3793-
this.consecutiveMistakeCount,
3794-
this.consecutiveMistakeLimit,
3795-
"no_tools_used",
3796-
this.apiConfiguration.apiProvider,
3797-
getModelId(this.apiConfiguration),
3798-
),
3784+
// Track consecutive mistake errors in telemetry via a named event.
3785+
// The reason is "unknown" because many paths increment consecutiveMistakeCount
3786+
// (no tool use, tool validation and tool execution failures) and none records its cause.
3787+
TelemetryService.instance.captureConsecutiveMistakeError(
3788+
this.taskId,
3789+
"unknown",
3790+
this.consecutiveMistakeCount,
3791+
this.consecutiveMistakeLimit,
37993792
)
38003793

38013794
const { response, text, images } = await this.ask(

0 commit comments

Comments
 (0)