Skip to content

Commit fa32ba8

Browse files
committed
feat(task): add run-state divergence comparison and metrics
1 parent 89a90f5 commit fa32ba8

8 files changed

Lines changed: 516 additions & 8 deletions

File tree

‎packages/types/src/telemetry.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,8 @@ export enum TelemetryEventName {
8686
TELEMETRY_SETTINGS_CHANGED = "Telemetry Settings Changed",
8787
MODEL_CACHE_EMPTY_RESPONSE = "Model Cache Empty Response",
8888
READ_FILE_LEGACY_FORMAT_USED = "Read File Legacy Format Used",
89+
RUN_STATE_DIVERGENCE = "Run State Divergence",
90+
RUN_STATE_KNOWN_RACE = "Run State Known Race",
8991
}
9092

9193
/**
@@ -221,6 +223,20 @@ export const rooCodeTelemetryEventSchema = z.discriminatedUnion("type", [
221223
]),
222224
properties: telemetryPropertiesSchema,
223225
}),
226+
z.object({
227+
type: z.enum([TelemetryEventName.RUN_STATE_DIVERGENCE, TelemetryEventName.RUN_STATE_KNOWN_RACE]),
228+
properties: z.object({
229+
...telemetryPropertiesSchema.shape,
230+
taskId: z.string(),
231+
kind: z.enum(["mismatch", "rejection"]),
232+
raceId: z.string().optional(),
233+
eventTag: z.string(),
234+
mismatched: z.array(z.string()),
235+
phase: z.string(),
236+
streamTag: z.string(),
237+
abort: z.boolean(),
238+
}),
239+
}),
224240
z.object({
225241
type: z.literal(TelemetryEventName.TELEMETRY_SETTINGS_CHANGED),
226242
properties: z.object({

‎src/__mocks__/vscode.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ export const env = {
107107
openExternal: () => Promise.resolve(),
108108
}
109109

110+
export const ExtensionMode = { Production: 1, Development: 2, Test: 3 }
110111
export const Uri = mockUri
111112
export const Range = mockRange
112113
export const Position = mockPosition

‎src/core/task/Task.ts‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,12 @@ import { shouldAddUserMessageToHistory } from "./messageCounting"
151151
import { type TaskExecutionContext } from "./providerHandoff"
152152
import { initialRunState, type Ask, type RunEvent, type RunState } from "./run-state/runState"
153153
import { applyRunEvent, type RunRejection } from "./run-state/runStateShadow"
154+
import {
155+
divergenceKey,
156+
findDivergences,
157+
isDivergenceCheckEnabled,
158+
reportDivergence,
159+
} from "./run-state/runStateDivergence"
154160

155161
const MAX_EXPONENTIAL_BACKOFF_SECONDS = 600 // 10 minutes
156162
const DEFAULT_USAGE_COLLECTION_TIMEOUT_MS = 5000 // 5 seconds
@@ -409,6 +415,8 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
409415

410416
runState: RunState = initialRunState
411417
readonly runRejections: RunRejection[] = []
418+
private readonly divergenceCheckEnabled: boolean
419+
private readonly reportedDivergences = new Set<string>()
412420

413421
// API
414422
apiConfiguration: ProviderSettings
@@ -734,6 +742,7 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
734742
)
735743
this.providerRef = new WeakRef(provider)
736744
this.globalStoragePath = provider.context.globalStorageUri.fsPath
745+
this.divergenceCheckEnabled = isDivergenceCheckEnabled(Package.releaseChannel, provider.context.extensionMode)
737746
this.diffViewProvider = new DiffViewProvider(this.cwd, this)
738747
this.enableCheckpoints = enableCheckpoints
739748
this.checkpointTimeout = checkpointTimeout
@@ -3458,7 +3467,26 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
34583467
}
34593468

34603469
private driveRunEvent(event: RunEvent): void {
3461-
this.runState = applyRunEvent(this.runState, event, this.runRejections)
3470+
const { state, rejected } = applyRunEvent(this.runState, event, this.runRejections)
3471+
this.runState = state
3472+
3473+
if (this.divergenceCheckEnabled) {
3474+
const fields = {
3475+
abort: this.abort,
3476+
abandoned: this.abandoned,
3477+
abortReason: this.abortReason,
3478+
isInitialized: this.isInitialized,
3479+
isStreaming: this.isStreaming,
3480+
didFinishAbortingStream: this.didFinishAbortingStream,
3481+
}
3482+
for (const divergence of findDivergences({ event, state, fields, rejected })) {
3483+
if (this.reportedDivergences.has(divergenceKey(divergence))) {
3484+
continue
3485+
}
3486+
this.reportedDivergences.add(divergenceKey(divergence))
3487+
reportDivergence(this.taskId, divergence)
3488+
}
3489+
}
34623490
}
34633491

34643492
public abortTask(isAbandoned = false): Promise<void> {

‎src/core/task/__tests__/Task.run-shadow.spec.ts‎

Lines changed: 75 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
// npx vitest run core/task/__tests__/Task.run-shadow.spec.ts
22

3+
import * as vscode from "vscode"
34
import { providerIdentifiers } from "@roo-code/types/provider-identifiers"
5+
import { TelemetryEventName } from "@roo-code/types"
46
import { TelemetryService } from "@roo-code/telemetry"
57

68
import { makeExtensionContext, makeUri } from "../../../test-utils/vscode"
@@ -11,10 +13,13 @@ import { initialRunState } from "../run-state/runState"
1113
vi.mock("../../../i18n", () => ({ t: (key: string) => key }))
1214
vi.mock("../../ignore/RooIgnoreController")
1315

14-
function makeTask(): Task {
16+
function makeTask(extensionMode = vscode.ExtensionMode.Production): Task {
1517
const provider = {
1618
log: vi.fn(),
17-
context: makeExtensionContext({ globalStorageUri: makeUri("/mock/global-storage") }),
19+
context: makeExtensionContext({
20+
globalStorageUri: makeUri("/mock/global-storage"),
21+
extensionMode,
22+
}),
1823
}
1924
// Double cast: the constructor reads only these provider members.
2025
return new Task({
@@ -129,3 +134,71 @@ describe("Task run-state write helpers", () => {
129134
expect(task.runRejections).toEqual([])
130135
})
131136
})
137+
138+
describe("Task run-state divergence report", () => {
139+
let warn: ReturnType<typeof vi.spyOn>
140+
let captureEvent: ReturnType<typeof vi.spyOn>
141+
142+
const divergenceMetrics = () =>
143+
captureEvent.mock.calls.filter(
144+
([name]: unknown[]) =>
145+
name === TelemetryEventName.RUN_STATE_DIVERGENCE || name === TelemetryEventName.RUN_STATE_KNOWN_RACE,
146+
)
147+
148+
const divergenceWarnings = () =>
149+
warn.mock.calls.filter(([message]: unknown[]) => message === "[RunState] divergence")
150+
151+
beforeEach(() => {
152+
if (!TelemetryService.hasInstance()) {
153+
TelemetryService.createInstance([])
154+
}
155+
warn = vi.spyOn(console, "warn").mockImplementation(() => {})
156+
captureEvent = vi.spyOn(TelemetryService.instance, "captureEvent").mockImplementation(() => {})
157+
})
158+
159+
afterEach(() => {
160+
vi.restoreAllMocks()
161+
})
162+
163+
it("logs and counts a rejected event once when the check is enabled", () => {
164+
const task = makeTask(vscode.ExtensionMode.Test)
165+
166+
task.requestAbandon()
167+
168+
expect(divergenceWarnings()).toHaveLength(1)
169+
expect(divergenceWarnings()[0][1]).toMatchObject({
170+
taskId: task.taskId,
171+
kind: "rejection",
172+
event: { tag: "abandonRequested" },
173+
state: initialRunState,
174+
mismatched: [],
175+
})
176+
expect(divergenceMetrics()).toEqual([
177+
[
178+
TelemetryEventName.RUN_STATE_DIVERGENCE,
179+
expect.objectContaining({ taskId: task.taskId, kind: "rejection", eventTag: "abandonRequested" }),
180+
],
181+
])
182+
})
183+
184+
it("reports a repeated rejection of the same event once per task", () => {
185+
const task = makeTask(vscode.ExtensionMode.Test)
186+
187+
task.requestAbandon()
188+
task.requestAbandon()
189+
190+
expect(task.runRejections).toHaveLength(2)
191+
expect(divergenceWarnings()).toHaveLength(1)
192+
expect(divergenceMetrics()).toHaveLength(1)
193+
})
194+
195+
it("stays silent for a production build on the stable channel", () => {
196+
const task = makeTask(vscode.ExtensionMode.Production)
197+
198+
task.requestAbandon()
199+
200+
expect(task.runRejections).toHaveLength(1)
201+
expect(divergenceWarnings()).toHaveLength(0)
202+
expect(divergenceMetrics()).toHaveLength(0)
203+
})
204+
})

0 commit comments

Comments
 (0)