Skip to content

Commit 8bcef34

Browse files
t3dotggclaude
andauthored
fix(server): analytics stop counting reconnects, retries, and dev servers (#24601)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 553203c commit 8bcef34

15 files changed

Lines changed: 330 additions & 87 deletions

‎apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1285,7 +1285,8 @@ it.effect("replays a thread.create receipt without changing the thread", () =>
12851285
const before = yield* projections.getThread(threadId);
12861286
const replayed = yield* orchestrator.dispatch(create);
12871287
assert.equal(replayed.sequence, original.sequence);
1288-
assert.deepEqual(replayed, original);
1288+
// Marked so client analytics do not count the retry again.
1289+
assert.deepEqual(replayed, { ...original, replayed: true });
12891290
assert.deepEqual(yield* projections.getThread(threadId), before);
12901291
assert.equal(yield* orchestrator.getThreadEventSequence(threadId), original.sequence);
12911292
}).pipe(Effect.provide(layerTest)),

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,8 @@ export type OrchestratorV2Error = typeof OrchestratorV2Error.Type;
295295
export interface OrchestratorV2DispatchResult {
296296
readonly sequence: number;
297297
readonly storedEvents: ReadonlyArray<OrchestrationV2StoredEvent>;
298+
/** The command id was already accepted; analytics skip it. */
299+
readonly replayed?: true;
298300
}
299301

300302
export interface OrchestratorV2Shape {
@@ -11791,6 +11793,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1179111793
return {
1179211794
sequence: receipt.resultSequence,
1179311795
storedEvents,
11796+
replayed: true,
1179411797
} satisfies OrchestratorV2DispatchResult;
1179511798
}
1179611799

@@ -11863,9 +11866,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1186311866
if (plan.events.length === 0 && plan.effects.length === 0) {
1186411867
// A settle that ended nothing still records its receipt: a replayed Stop
1186511868
// effect then finds it instead of settling work that appeared since.
11866-
const resultSequence = yield* Effect.gen(function* () {
11869+
const { resultSequence, inserted } = yield* Effect.gen(function* () {
1186711870
const sequence = yield* eventSink.latestSequence({ threadId: commandThreadId(command) });
11868-
yield* commandReceipts.insertIfAbsent({
11871+
const inserted = yield* commandReceipts.insertIfAbsent({
1186911872
commandId: command.commandId,
1187011873
threadId: commandThreadId(command),
1187111874
commandType: command.type,
@@ -11874,7 +11877,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1187411877
status: "accepted",
1187511878
error: null,
1187611879
});
11877-
return sequence;
11880+
return { resultSequence: sequence, inserted };
1187811881
}).pipe(
1187911882
Effect.mapError(
1188011883
(cause) =>
@@ -11885,7 +11888,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1188511888
}),
1188611889
),
1188711890
);
11888-
return { sequence: resultSequence, storedEvents: [] } satisfies OrchestratorV2DispatchResult;
11891+
return {
11892+
sequence: resultSequence,
11893+
storedEvents: [],
11894+
...(inserted ? {} : { replayed: true as const }),
11895+
} satisfies OrchestratorV2DispatchResult;
1188911896
}
1189011897
const acceptedAt = plan.events.at(-1)?.occurredAt ?? (yield* DateTime.now);
1189111898
const committed = yield* eventSink
@@ -11936,6 +11943,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1193611943
return {
1193711944
sequence: committed.receipt.resultSequence,
1193811945
storedEvents: committed.storedEvents,
11946+
...(committed.committed ? {} : { replayed: true as const }),
1193911947
} satisfies OrchestratorV2DispatchResult;
1194011948
});
1194111949

‎apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,12 @@ it.effect("records accepted billed turn usage once without billing the context w
233233
modelSelection,
234234
runtimeMode: "full-access" as const,
235235
interactionMode: "default" as const,
236+
origin: {
237+
createdBy: "agent" as const,
238+
creationSource: "provider" as const,
239+
scheduled: false,
240+
subagent: true,
241+
},
236242
},
237243
event: { type: "provider_turn.updated" as const, driver: CODEX_DRIVER, providerTurn },
238244
};
@@ -271,6 +277,10 @@ it.effect("records accepted billed turn usage once without billing the context w
271277
mixedModels: false,
272278
runtimeMode: "full-access",
273279
interactionMode: "default",
280+
createdBy: "agent",
281+
creationSource: "provider",
282+
scheduled: false,
283+
subagent: true,
274284
durationMs: 120,
275285
});
276286
}).pipe(Effect.provide(layerTest.pipe(Layer.provide(layerAnalytics))));

‎apps/server/src/orchestration-v2/ProviderEventIngestor.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ import {
99
type OrchestrationV2Subagent,
1010
type OrchestrationV2ThreadProjection,
1111
type ModelSelection,
12+
type OrchestrationV2Actor,
13+
type OrchestrationV2CreationSource,
1214
type RuntimeMode,
1315
type ProviderInteractionMode,
1416
ProviderInstanceId,
@@ -77,6 +79,19 @@ export interface ProviderTurnAnalyticsContext {
7779
readonly modelSelection: ModelSelection;
7880
readonly runtimeMode?: RuntimeMode;
7981
readonly interactionMode?: ProviderInteractionMode;
82+
readonly origin?: ProviderTurnOrigin;
83+
}
84+
85+
/**
86+
* Where the run's starting message came from. Most turns start without a
87+
* person (provider continuations, agent sends, delegated subagents), so
88+
* reports need this to tell user turns apart.
89+
*/
90+
export interface ProviderTurnOrigin {
91+
readonly createdBy: OrchestrationV2Actor;
92+
readonly creationSource: OrchestrationV2CreationSource;
93+
readonly scheduled: boolean;
94+
readonly subagent: boolean;
8095
}
8196

8297
export class ProviderTurnAnalytics extends Context.Reference<{
@@ -126,6 +141,7 @@ function providerTurnAnalyticsProperties(input: {
126141
...(effort ? { effort } : {}),
127142
...(input.context?.runtimeMode ? { runtimeMode: input.context.runtimeMode } : {}),
128143
...(input.context?.interactionMode ? { interactionMode: input.context.interactionMode } : {}),
144+
...input.context?.origin,
129145
...(input.providerTurn.startedAt && input.providerTurn.completedAt
130146
? {
131147
durationMs: Math.max(

‎apps/server/src/orchestration-v2/RunExecutionService.test.ts‎

Lines changed: 35 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,11 @@ import * as RunFinalizationService from "./RunFinalizationService.ts";
5050
import * as TurnItemPositionStore from "./TurnItemPositionStore.ts";
5151

5252
const driver = ProviderDriverKind.make("codex");
53+
const rootAppThread = (id: ThreadId) =>
54+
({
55+
id,
56+
lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: id },
57+
}) as OrchestrationV2AppThread;
5358

5459
const layerTurnItemPositionsTest = Layer.mock(TurnItemPositionStore.TurnItemPositionStoreV2)({
5560
allocate: () => Effect.succeed(1),
@@ -775,7 +780,7 @@ it.effect("rechecks run ownership immediately before calling the provider", () =
775780

776781
yield* runExecution.startRootRun({
777782
commandId: CommandId.make("command:run-execution-start-guard"),
778-
appThread: { id: threadId } as OrchestrationV2AppThread,
783+
appThread: rootAppThread(threadId),
779784
providerSessionId,
780785
session,
781786
run,
@@ -830,7 +835,7 @@ it.effect("passes the thread's MCP app context to the provider under a safe key"
830835

831836
yield* runExecution.startRootRun({
832837
commandId: CommandId.make("command:run-execution-app-context"),
833-
appThread: { id: threadId } as OrchestrationV2AppThread,
838+
appThread: rootAppThread(threadId),
834839
providerSessionId: ProviderSessionId.make("session:run-execution-app-context"),
835840
session,
836841
run: {
@@ -934,7 +939,7 @@ it.effect("fails the run when its ownership check cannot be read before calling
934939
const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
935940
yield* runExecution.startRootRun({
936941
commandId: CommandId.make("command:run-execution-start-guard-read"),
937-
appThread: { id: threadId } as OrchestrationV2AppThread,
942+
appThread: rootAppThread(threadId),
938943
providerSessionId: ProviderSessionId.make("session:run-execution-start-guard-read"),
939944
session: {
940945
events: Stream.never,
@@ -1029,7 +1034,7 @@ it.effect(
10291034
const attemptId = RunAttemptId.make(`attempt:compact-routing:${index}`);
10301035
yield* runExecution.startRootRun({
10311036
commandId: CommandId.make(`command:compact-routing:${index}`),
1032-
appThread: { id: threadId } as OrchestrationV2AppThread,
1037+
appThread: rootAppThread(threadId),
10331038
providerSessionId: ProviderSessionId.make(`session:compact-routing:${index}`),
10341039
session: {
10351040
events: Stream.never,
@@ -1145,7 +1150,7 @@ it.effect("starts the provider when checkpoint baseline capture fails", () =>
11451150
const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
11461151
yield* runExecution.startRootRun({
11471152
commandId: CommandId.make("command:run-execution-baseline-failure"),
1148-
appThread: { id: threadId } as OrchestrationV2AppThread,
1153+
appThread: rootAppThread(threadId),
11491154
providerSessionId,
11501155
session: {
11511156
events: Stream.never,
@@ -1283,7 +1288,7 @@ it.effect.each(["failure", "interruption", "stale-attempt", "start-guard"] as co
12831288
const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
12841289
yield* runExecution.startRootRun({
12851290
commandId: CommandId.make("command:run-execution-settings-failure"),
1286-
appThread: { id: threadId } as OrchestrationV2AppThread,
1291+
appThread: rootAppThread(threadId),
12871292
providerSessionId,
12881293
session: {
12891294
events: Stream.never,
@@ -1399,6 +1404,7 @@ it.effect("ingests new member threads of an owned child after the root turn term
13991404
const subagentNodeId = NodeId.make("node:run-execution-late-child:subagent");
14001405
const childMessageIngested = yield* Deferred.make<void>();
14011406
const order = yield* Ref.make<ReadonlyArray<string>>([]);
1407+
const origins = yield* Ref.make<ReadonlyArray<readonly [string, unknown]>>([]);
14021408
const layerTest = RunExecutionService.layer.pipe(
14031409
Layer.provide(
14041410
Layer.mergeAll(
@@ -1424,6 +1430,11 @@ it.effect("ingests new member threads of an owned child after the root turn term
14241430
Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
14251431
ingestNormalized: (input) =>
14261432
Effect.gen(function* () {
1433+
const eventType = input.event.type;
1434+
yield* Ref.update(origins, (current) => [
1435+
...current,
1436+
[eventType, input.analyticsContext?.origin] as const,
1437+
]);
14271438
if (
14281439
input.event.type === "app_thread.created" &&
14291440
input.event.appThread.id === memberThreadId
@@ -1559,7 +1570,7 @@ it.effect("ingests new member threads of an owned child after the root turn term
15591570
const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
15601571
yield* runExecution.startRootRun({
15611572
commandId: CommandId.make("command:run-execution-late-child"),
1562-
appThread: { id: threadId } as OrchestrationV2AppThread,
1573+
appThread: rootAppThread(threadId),
15631574
providerSessionId,
15641575
session: {
15651576
events: Stream.fromIterable(events),
@@ -1612,6 +1623,19 @@ it.effect("ingests new member threads of an owned child after the root turn term
16121623
);
16131624
assert.isTrue(Option.isSome(observed), "child message was not ingested after root terminal");
16141625
assert.deepEqual(yield* Ref.get(order), ["root-finalized", "member-thread", "child-message"]);
1626+
// The run's message started the root; the parent agent started the child's turn.
1627+
const recorded = yield* Ref.get(origins);
1628+
assert.deepEqual(recorded[0], [
1629+
"app_thread.created",
1630+
{ createdBy: "user", creationSource: "web", scheduled: false, subagent: false },
1631+
]);
1632+
assert.deepEqual(
1633+
recorded.find(([type]) => type === "provider_turn.updated"),
1634+
[
1635+
"provider_turn.updated",
1636+
{ createdBy: "agent", creationSource: "provider", scheduled: false, subagent: true },
1637+
],
1638+
);
16151639
}),
16161640
);
16171641

@@ -1704,7 +1728,7 @@ it.effect("ends a nested native subagent and its thread when the root run is int
17041728
const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
17051729
yield* runExecution.startRootRun({
17061730
commandId: CommandId.make(`command:${key}`),
1707-
appThread: { id: threadId } as OrchestrationV2AppThread,
1731+
appThread: rootAppThread(threadId),
17081732
providerSessionId: ProviderSessionId.make(`session:${key}`),
17091733
session: {
17101734
events: Stream.fromIterable(events),
@@ -2031,7 +2055,7 @@ it.effect(
20312055
const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
20322056
yield* runExecution.startRootRun({
20332057
commandId: CommandId.make("command:subagent-interrupt-cascade"),
2034-
appThread: { id: ids.threadId } as OrchestrationV2AppThread,
2058+
appThread: rootAppThread(ids.threadId),
20352059
providerSessionId: ProviderSessionId.make("session:subagent-interrupt-cascade"),
20362060
session: {
20372061
events: Stream.empty,
@@ -2816,7 +2840,7 @@ function captureRootRunTermination(input: {
28162840
const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
28172841
const start = runExecution.startRootRun({
28182842
commandId: CommandId.make(`command:${input.key}`),
2819-
appThread: { id: ids.threadId } as OrchestrationV2AppThread,
2843+
appThread: rootAppThread(ids.threadId),
28202844
providerSessionId: ProviderSessionId.make(`session:${input.key}`),
28212845
session: {
28222846
events: Stream.empty,
@@ -3312,7 +3336,7 @@ function runBackgroundItemScenario(
33123336
const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
33133337
yield* runExecution.startRootRun({
33143338
commandId: CommandId.make(`command:${key}`),
3315-
appThread: { id: ids.threadId } as OrchestrationV2AppThread,
3339+
appThread: rootAppThread(ids.threadId),
33163340
providerSessionId: ProviderSessionId.make(`session:${key}`),
33173341
session: {
33183342
events: Stream.empty,

‎apps/server/src/orchestration-v2/RunExecutionService.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,14 @@ function isSettledSubagentStatus(status: OrchestrationV2Subagent["status"]): boo
9797
return !isOrchestrationV2WorkActive(status);
9898
}
9999

100+
/** A provider-native subagent's turns arrive on its parent's run, but the parent agent started them. */
101+
const NATIVE_SUBAGENT_TURN_ORIGIN: ProviderEventIngestor.ProviderTurnOrigin = {
102+
createdBy: "agent",
103+
creationSource: "provider",
104+
scheduled: false,
105+
subagent: true,
106+
};
107+
100108
const backgroundCapableTurnItemTypes: ReadonlySet<OrchestrationV2TurnItem["type"]> = new Set([
101109
"command_execution",
102110
"dynamic_tool",
@@ -883,6 +891,12 @@ export const layer: Layer.Layer<
883891

884892
return RunExecutionServiceV2.of({
885893
startRootRun: Effect.fn("RunExecutionService.startRootRun")(function* (input) {
894+
const rootTurnOrigin: ProviderEventIngestor.ProviderTurnOrigin = {
895+
createdBy: input.message.createdBy,
896+
creationSource: input.message.creationSource,
897+
scheduled: input.message.scheduledTaskId !== undefined,
898+
subagent: input.appThread.lineage.relationshipToParent === "subagent",
899+
};
886900
// Startup failure and stream shutdown can report the same attempt.
887901
const refreshAfterTurn = yield* Effect.cached(
888902
finalizationObserver.refreshAfterTurn(input.appThread.projectId).pipe(
@@ -1375,6 +1389,11 @@ export const layer: Layer.Layer<
13751389
modelSelection: input.modelSelection,
13761390
runtimeMode: input.runtimePolicy.runtimeMode,
13771391
interactionMode: input.runtimePolicy.interactionMode,
1392+
origin:
1393+
event.type === "provider_turn.updated" &&
1394+
event.providerTurn.providerThreadId !== input.providerThread.id
1395+
? NATIVE_SUBAGENT_TURN_ORIGIN
1396+
: rootTurnOrigin,
13781397
},
13791398
providerSessionId: input.providerSessionId,
13801399
providerInstanceId: input.run.providerInstanceId,

‎apps/server/src/orchestration-v2/ThreadLaunchService.test.ts‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1597,6 +1597,73 @@ it.effect("does not let a failing same-command caller strand a concurrent durabl
15971597
}),
15981598
);
15991599

1600+
it.effect("reports one concurrent same-command launch as new and the rest as resumed", () =>
1601+
Effect.gen(function* () {
1602+
// Both launches read no receipt before either one claims the thread.
1603+
const arrived = yield* Ref.make(0);
1604+
const bothRead = yield* Deferred.make<void>();
1605+
const harness = makeHarness({
1606+
managedFolders: Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
1607+
namedProjectsRoot: "/projects",
1608+
isScratchProject: () => Effect.succeed(false),
1609+
folderForThread: () =>
1610+
Ref.updateAndGet(arrived, (count) => count + 1).pipe(
1611+
Effect.tap((count) =>
1612+
count === 2 ? Deferred.succeed(bothRead, undefined) : Effect.void,
1613+
),
1614+
Effect.andThen(Deferred.await(bothRead)),
1615+
Effect.as(Option.none()),
1616+
),
1617+
}),
1618+
});
1619+
yield* Effect.gen(function* () {
1620+
const launches = yield* ThreadLaunch.ThreadLaunchService;
1621+
const input = launchInput({
1622+
command: "command:launch:concurrent-resumed",
1623+
thread: "thread:launch:concurrent-resumed",
1624+
});
1625+
const results = yield* Effect.all([launches.launch(input), launches.launch(input)], {
1626+
concurrency: "unbounded",
1627+
});
1628+
assert.deepEqual(results.map((result) => result.resumed).sort(), [false, true]);
1629+
}).pipe(Effect.provide(harness.layer));
1630+
}),
1631+
);
1632+
1633+
it.effect(
1634+
"does not report a retry as resumed when the earlier attempt only claimed the thread",
1635+
() =>
1636+
Effect.gen(function* () {
1637+
const harness = makeHarness();
1638+
yield* Effect.gen(function* () {
1639+
const launches = yield* ThreadLaunch.ThreadLaunchService;
1640+
const threads = yield* ThreadManagement.ThreadManagementService;
1641+
const input = launchInput({
1642+
command: "command:launch:claimed-only",
1643+
thread: "thread:launch:claimed-only",
1644+
message: "Counted once",
1645+
});
1646+
yield* threads.dispatch({
1647+
type: "thread.create",
1648+
commandId: input.commandId,
1649+
threadId: input.threadId,
1650+
projectId: input.projectId,
1651+
title: input.title,
1652+
modelSelection: input.modelSelection,
1653+
runtimeMode: input.runtimeMode,
1654+
interactionMode: input.interactionMode,
1655+
branch: null,
1656+
worktreePath: null,
1657+
createdBy: input.createdBy,
1658+
creationSource: input.creationSource,
1659+
});
1660+
const retried = yield* launches.launch(input);
1661+
assert.isFalse(retried.resumed);
1662+
assert.isTrue((yield* launches.launch(input)).resumed);
1663+
}).pipe(Effect.provide(harness.layer));
1664+
}),
1665+
);
1666+
16001667
it.effect("schedules an accepted preparing message exactly once across concurrent retries", () =>
16011668
Effect.gen(function* () {
16021669
const setupEntered = yield* Deferred.make<void>();

0 commit comments

Comments
 (0)