Skip to content

Commit f8e4834

Browse files
feat(server): export how long runs take and how they end (#22094)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 6f6f68a commit f8e4834

4 files changed

Lines changed: 454 additions & 20 deletions

File tree

‎apps/server/src/observability/Metrics.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,18 @@ export const serverEventLoopDelayMax = Metric.gauge("t3_server_event_loop_delay_
6363
attributes: { unit: "ms" },
6464
});
6565

66+
export const runsTotal = Metric.counter("t3_runs_total", {
67+
description: "Runs that finished, by outcome. A later rollback does not count again.",
68+
});
69+
70+
export const runDuration = Metric.timer("t3_run_duration", {
71+
description: "Wall time of a run, from when it started to its terminal status.",
72+
});
73+
74+
export const runQueueWait = Metric.timer("t3_run_queue_wait", {
75+
description: "Time a run waited between being requested and starting.",
76+
});
77+
6678
export const gitCommandsTotal = Metric.counter("t3_git_commands_total", {
6779
description: "Total git commands executed by the server runtime.",
6880
});

‎apps/server/src/orchestration-v2/EventSink.ts‎

Lines changed: 45 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import * as SqlClient from "effect/sql/SqlClient";
2626
import { replayAndBufferProjectedLiveEvents } from "./LiveStreamBudget.ts";
2727
import type { UnsequencedProjectEvent } from "../persistence/OrchestrationEventStore.ts";
2828
import { projectDomainEventForWire } from "./WireProjection.ts";
29+
import { makeRunOutcomeRecorder, type RunOutcome } from "./RunMetrics.ts";
2930

3031
import * as CommandReceiptStore from "./CommandReceiptStore.ts";
3132
import * as EffectOutbox from "./EffectOutbox.ts";
@@ -243,8 +244,23 @@ const layerBase: Layer.Layer<
243244
);
244245
}
245246
});
246-
const publishStoredEvents = (events: ReadonlyArray<OrchestrationV2StoredEvent>) =>
247-
eventStore.publishCommitted(events).pipe(Effect.andThen(publishLiveEvents(events)));
247+
const runOutcomes = makeRunOutcomeRecorder((threadId) =>
248+
// One payload row; unlike the shell, it still resolves a deleted child thread.
249+
projectionStore
250+
.getThread(threadId)
251+
.pipe(Effect.map((thread) => thread.lineage.relationshipToParent === "subagent")),
252+
);
253+
// Recorded before publication, so an interrupted publish cannot lose a committed outcome.
254+
const publishStoredEvents = (
255+
events: ReadonlyArray<OrchestrationV2StoredEvent>,
256+
outcomes: ReadonlyArray<RunOutcome>,
257+
) =>
258+
runOutcomes
259+
.record(outcomes)
260+
.pipe(
261+
Effect.andThen(eventStore.publishCommitted(events)),
262+
Effect.andThen(publishLiveEvents(events)),
263+
);
248264

249265
// Transactions commit one at a time, but each writer publishes after its
250266
// commit. If a writer is descheduled in between, a later commit reaches
@@ -373,6 +389,8 @@ const layerBase: Layer.Layer<
373389
updated_at = excluded.updated_at
374390
`;
375391
}
392+
// Read lineage here, on the connection this transaction holds; see makeRunOutcomeRecorder.
393+
return yield* runOutcomes.prepare(storedEvents);
376394
});
377395

378396
const writeEffect = Effect.fn("orchestrationV2.EventSink.write")(function* (
@@ -384,7 +402,7 @@ const layerBase: Layer.Layer<
384402
"orchestration_v2.thread_id": input.events[0]?.threadId ?? null,
385403
});
386404

387-
return yield* commitThenPublish(
405+
const { committed } = yield* commitThenPublish(
388406
Effect.gen(function* () {
389407
// Check ownership in the same transaction that claims the native session.
390408
const imported = input.unlessNativeThreadOwned;
@@ -398,7 +416,7 @@ const layerBase: Layer.Layer<
398416
AND json_extract(payload_json, '$.nativeThreadRef.nativeId') = ${imported.nativeThreadRef.nativeId}
399417
LIMIT 1
400418
`;
401-
if (owners.length > 0) return [];
419+
if (owners.length > 0) return { committed: [], outcomes: [] };
402420
}
403421
// Checkpoint capture can finish after a cohort writer reads its parent run.
404422
// Merge in this transaction so the event log and projection keep that completion.
@@ -433,18 +451,19 @@ const layerBase: Layer.Layer<
433451
...(input.commandId === undefined ? {} : { commandId: input.commandId }),
434452
events: normalized,
435453
});
436-
yield* applyStoredEvents(committed);
454+
const outcomes = yield* applyStoredEvents(committed);
437455
yield* effectOutbox.enqueue(input.effects);
438-
return committed;
456+
return { committed, outcomes };
439457
}).pipe(Effect.withTracerEnabled(false)),
440-
(storedEvents) =>
458+
({ committed, outcomes }) =>
441459
Effect.gen(function* () {
442460
if (input.effects.length > 0) {
443461
yield* effectOutbox.notifyAvailable(input.effects.length);
444462
}
445-
yield* publishStoredEvents(storedEvents);
463+
yield* publishStoredEvents(committed, outcomes);
446464
}),
447465
);
466+
return committed;
448467
});
449468

450469
const writeIfRunCurrentEffect = Effect.fn("orchestrationV2.EventSink.writeIfRunCurrent")(
@@ -491,17 +510,17 @@ const layerBase: Layer.Layer<
491510
...(input.commandId === undefined ? {} : { commandId: input.commandId }),
492511
events: normalized,
493512
});
494-
yield* applyStoredEvents(storedEvents);
513+
const outcomes = yield* applyStoredEvents(storedEvents);
495514
yield* effectOutbox.enqueue(input.effects ?? []);
496-
return { committed: true as const, storedEvents };
515+
return { committed: true as const, storedEvents, outcomes };
497516
}),
498517
(result) =>
499518
Effect.gen(function* () {
500519
if (!result.committed) return;
501520
if (input.effects !== undefined && input.effects.length > 0) {
502521
yield* effectOutbox.notifyAvailable(input.effects.length);
503522
}
504-
yield* publishStoredEvents(result.storedEvents);
523+
yield* publishStoredEvents(result.storedEvents, result.outcomes);
505524
}),
506525
);
507526
},
@@ -556,10 +575,13 @@ const layerBase: Layer.Layer<
556575
...(input.commandId === undefined ? {} : { commandId: input.commandId }),
557576
events: normalized,
558577
});
559-
yield* applyStoredEvents(storedEvents);
560-
return { committed: true as const, storedEvents };
578+
const outcomes = yield* applyStoredEvents(storedEvents);
579+
return { committed: true as const, storedEvents, outcomes };
561580
}),
562-
(result) => (result.committed ? publishStoredEvents(result.storedEvents) : Effect.void),
581+
(result) =>
582+
result.committed
583+
? publishStoredEvents(result.storedEvents, result.outcomes)
584+
: Effect.void,
563585
);
564586
});
565587

@@ -589,10 +611,13 @@ const layerBase: Layer.Layer<
589611

590612
const normalized = yield* normalizeEvents(input.events);
591613
const storedEvents = yield* eventStore.append({ events: normalized });
592-
yield* applyStoredEvents(storedEvents);
593-
return { committed: true as const, storedEvents };
614+
const outcomes = yield* applyStoredEvents(storedEvents);
615+
return { committed: true as const, storedEvents, outcomes };
594616
}),
595-
(result) => (result.committed ? publishStoredEvents(result.storedEvents) : Effect.void),
617+
(result) =>
618+
result.committed
619+
? publishStoredEvents(result.storedEvents, result.outcomes)
620+
: Effect.void,
596621
);
597622
});
598623

@@ -642,7 +667,7 @@ const layerBase: Layer.Layer<
642667
new Error(`Command ${input.commandId} produced no orchestration events.`),
643668
);
644669
}
645-
yield* applyStoredEvents(storedEvents);
670+
const outcomes = yield* applyStoredEvents(storedEvents);
646671
yield* effectOutbox.enqueue(input.effects);
647672
const receipt: CommandReceiptStore.CommandReceiptV2 = {
648673
commandId: input.commandId,
@@ -661,15 +686,15 @@ const layerBase: Layer.Layer<
661686
threadId: input.threadId,
662687
...input.cancelUnsettledEffects,
663688
});
664-
return { receipt, storedEvents, committed: true as const, cancelledEffectIds };
689+
return { receipt, storedEvents, outcomes, committed: true as const, cancelledEffectIds };
665690
}),
666691
(result) =>
667692
Effect.gen(function* () {
668693
yield* effectOutbox.signalCancellations(result.cancelledEffectIds);
669694
if (result.committed && input.effects.length > 0) {
670695
yield* effectOutbox.notifyAvailable(input.effects.length);
671696
}
672-
if (result.committed) yield* publishStoredEvents(result.storedEvents);
697+
if (result.committed) yield* publishStoredEvents(result.storedEvents, result.outcomes);
673698
}),
674699
);
675700
return {

0 commit comments

Comments
 (0)