Skip to content

Commit a61acfd

Browse files
only21milclaude
andcommitted
fix(server): queued messages still start after a failed startup event read
The orchestrator's startup reactors subscribe to run, provider-thread and subagent updates once and fork the stream detached. Subscribing reads the event table, so a failed read there ended the reactor for the life of the server: queued messages stopped promoting and delegated children stopped finalizing until a restart. Each reactor now resubscribes with a capped backoff, resuming after the last event it received. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent d53873b commit a61acfd

3 files changed

Lines changed: 184 additions & 25 deletions

File tree

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
import { assert, it } from "@effect/vitest";
2+
import {
3+
CommandId,
4+
EventId,
5+
MessageId,
6+
ProjectId,
7+
ProviderDriverKind,
8+
ProviderInstanceId,
9+
ThreadId,
10+
} from "@t3tools/contracts";
11+
import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
12+
import * as DateTime from "effect/DateTime";
13+
import * as Deferred from "effect/Deferred";
14+
import * as Effect from "effect/Effect";
15+
import * as Fiber from "effect/Fiber";
16+
import * as Stream from "effect/Stream";
17+
import * as TestClock from "effect/testing/TestClock";
18+
19+
import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
20+
import * as EventSink from "./EventSink.ts";
21+
import * as Orchestrator from "./Orchestrator.ts";
22+
import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts";
23+
import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts";
24+
25+
const instanceId = ProviderInstanceId.make("codex");
26+
const adapter = {
27+
instanceId,
28+
driver: ProviderDriverKind.make("codex"),
29+
getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2),
30+
planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" as const }),
31+
openSession: () => Effect.die("Runs here never reach a provider"),
32+
} as ProviderAdapter.ProviderAdapterV2["Service"];
33+
34+
it.effect("promotes the queue after the run-update subscription fails to start", () =>
35+
Effect.gen(function* () {
36+
const subscriptionFailed = yield* Deferred.make<void>();
37+
let failNextSubscription = true;
38+
const eventSink = (sink: EventSink.EventSinkV2Shape): EventSink.EventSinkV2Shape => ({
39+
...sink,
40+
stream: (input) => {
41+
if (input?.eventType !== "run.updated" || !failNextSubscription) return sink.stream(input);
42+
failNextSubscription = false;
43+
return Stream.fail(
44+
new EventSink.EventSinkStreamError({ cause: "database is locked" }),
45+
).pipe(Stream.ensuring(Deferred.succeed(subscriptionFailed, undefined)));
46+
},
47+
});
48+
yield* Effect.gen(function* () {
49+
const orchestrator = yield* Orchestrator.OrchestratorV2;
50+
const sink = yield* EventSink.EventSinkV2;
51+
const threadId = ThreadId.make("thread:reactor-resubscribe");
52+
yield* Deferred.await(subscriptionFailed);
53+
yield* orchestrator.dispatch({
54+
type: "thread.create",
55+
commandId: CommandId.make("create"),
56+
threadId,
57+
projectId: ProjectId.make("project:reactor-resubscribe"),
58+
title: "Reactor resubscribe",
59+
modelSelection: { instanceId, model: "gpt-5.1-codex" },
60+
runtimeMode: "full-access",
61+
interactionMode: "default",
62+
branch: null,
63+
worktreePath: null,
64+
createdBy: "user",
65+
creationSource: "web",
66+
});
67+
for (const [text, type] of [
68+
["active", "start_immediately"],
69+
["queued", "queue_after_active"],
70+
] as const) {
71+
yield* orchestrator.dispatch({
72+
type: "message.dispatch",
73+
commandId: CommandId.make(`send:${text}`),
74+
threadId,
75+
messageId: MessageId.make(`message:${text}`),
76+
text,
77+
attachments: [],
78+
dispatchMode: { type },
79+
createdBy: "user",
80+
creationSource: "web",
81+
});
82+
}
83+
const [active, queued] = (yield* orchestrator.getThreadProjection(threadId)).runs;
84+
assert.equal(queued?.status, "queued");
85+
const promoted = yield* orchestrator.streamDomainEvents.pipe(
86+
Stream.filter(
87+
(event) =>
88+
event.type === "run.updated" &&
89+
event.runId === queued!.id &&
90+
event.payload.status !== "queued",
91+
),
92+
Stream.take(1),
93+
Stream.runDrain,
94+
Effect.forkScoped,
95+
);
96+
const now = yield* DateTime.now;
97+
yield* sink.write({
98+
events: [
99+
{
100+
id: EventId.make("event:reactor-resubscribe:completed"),
101+
type: "run.updated",
102+
threadId,
103+
runId: active!.id,
104+
occurredAt: now,
105+
payload: { ...active!, status: "completed", startedAt: now, completedAt: now },
106+
},
107+
],
108+
});
109+
// The completion landed while no reactor was subscribed; the retry replays it.
110+
yield* TestClock.adjust("1 second");
111+
yield* Fiber.join(promoted);
112+
const after = yield* orchestrator.getThreadProjection(threadId);
113+
assert.notEqual(after.runs.find((run) => run.id === queued!.id)?.status, "queued");
114+
}).pipe(
115+
Effect.provide(
116+
ProviderReplayHarness.layerWithRegistry(
117+
{ name: "reactor-resubscribe" },
118+
ProviderAdapterRegistry.layerFromAdapters([adapter]),
119+
{ runEffectWorker: false, eventSink },
120+
),
121+
),
122+
);
123+
}).pipe(Effect.scoped),
124+
);

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 48 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -71,13 +71,15 @@ import {
7171
} from "@t3tools/shared/orchestrationV2PendingBackgroundWork";
7272
import * as Context from "effect/Context";
7373
import * as DateTime from "effect/DateTime";
74+
import * as Duration from "effect/Duration";
7475
import * as Effect from "effect/Effect";
7576
import * as Exit from "effect/Exit";
7677
import * as FileSystem from "effect/FileSystem";
7778
import * as Path from "effect/Path";
7879
import * as Layer from "effect/Layer";
7980
import * as Option from "effect/Option";
8081
import * as Ref from "effect/Ref";
82+
import * as Schedule from "effect/Schedule";
8183
import * as Schema from "effect/Schema";
8284
import * as Stream from "effect/Stream";
8385

@@ -12000,27 +12002,52 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1200012002
// below. Replaying the full event table on every server start delays live
1200112003
// queue promotion in proportion to the lifetime size of the database.
1200212004
const terminalEventsAfterSequence = yield* eventSink.latestSequence().pipe(Effect.orDie);
12003-
// Queue promotion can wait on a provider or a thread lock. Subscribe to run
12004-
// updates before buffering so that wait never retains unrelated tool bodies.
12005-
yield* eventSink
12006-
.stream({ afterSequence: terminalEventsAfterSequence, eventType: "run.updated" })
12007-
.pipe(
12008-
Stream.filter(
12009-
(stored) =>
12010-
stored.event.type === "run.updated" &&
12011-
!String(stored.commandId).startsWith("command:runtime-reconcile:") &&
12012-
((stored.event.payload.delegatedTaskId !== undefined &&
12013-
(stored.event.payload.status === "queued" ||
12014-
stored.event.payload.status === "starting")) ||
12015-
stored.event.payload.status === "completed" ||
12016-
stored.event.payload.status === "interrupted" ||
12017-
stored.event.payload.status === "failed" ||
12018-
stored.event.payload.status === "cancelled" ||
12019-
stored.event.payload.status === "rolled_back"),
12005+
// The reactors below run for the life of the server. Subscribing reads the
12006+
// event table, and a failed read would end a reactor for good, so each one
12007+
// resubscribes after the last event it received.
12008+
const reactorEvents = (eventType: OrchestrationV2DomainEvent["type"]) => {
12009+
let afterSequence = terminalEventsAfterSequence;
12010+
return Stream.suspend(() => eventSink.stream({ afterSequence, eventType })).pipe(
12011+
Stream.tap((stored) =>
12012+
Effect.sync(() => {
12013+
afterSequence = stored.sequence;
12014+
}),
12015+
),
12016+
Stream.tapError((cause) =>
12017+
Effect.logWarning("Resubscribing to V2 events after a failed read", {
12018+
eventType,
12019+
afterSequence,
12020+
cause,
12021+
}),
12022+
),
12023+
Stream.retry(
12024+
Schedule.exponential("1 second").pipe(
12025+
Schedule.modifyDelay(({ duration }) =>
12026+
Effect.succeed(Duration.min(duration, Duration.seconds(30))),
12027+
),
12028+
),
1202012029
),
12021-
Stream.runForEach(handleRunUpdate),
12022-
Effect.forkDetach,
1202312030
);
12031+
};
12032+
// Queue promotion can wait on a provider or a thread lock. Subscribe to run
12033+
// updates before buffering so that wait never retains unrelated tool bodies.
12034+
yield* reactorEvents("run.updated").pipe(
12035+
Stream.filter(
12036+
(stored) =>
12037+
stored.event.type === "run.updated" &&
12038+
!String(stored.commandId).startsWith("command:runtime-reconcile:") &&
12039+
((stored.event.payload.delegatedTaskId !== undefined &&
12040+
(stored.event.payload.status === "queued" ||
12041+
stored.event.payload.status === "starting")) ||
12042+
stored.event.payload.status === "completed" ||
12043+
stored.event.payload.status === "interrupted" ||
12044+
stored.event.payload.status === "failed" ||
12045+
stored.event.payload.status === "cancelled" ||
12046+
stored.event.payload.status === "rolled_back"),
12047+
),
12048+
Stream.runForEach(handleRunUpdate),
12049+
Effect.forkDetach,
12050+
);
1202412051

1202512052
// A child whose run ends while its background tasks or its own subagents still run has no
1202612053
// result yet. When that work settles there is no run update to finalize it, so these events
@@ -12038,11 +12065,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1203812065
return providerThreadsWithBackgroundWork.delete(event.payload.id);
1203912066
};
1204012067
yield* Stream.merge(
12041-
eventSink.stream({
12042-
afterSequence: terminalEventsAfterSequence,
12043-
eventType: "provider-thread.updated",
12044-
}),
12045-
eventSink.stream({ afterSequence: terminalEventsAfterSequence, eventType: "subagent.updated" }),
12068+
reactorEvents("provider-thread.updated"),
12069+
reactorEvents("subagent.updated"),
1204612070
).pipe(
1204712071
Stream.filter(settlesChildWork),
1204812072
Stream.runForEach((stored) =>

‎apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,7 @@ export function layerWithRegistry<Error>(
270270
readonly runContinuationWorker?: boolean;
271271
readonly recoverOnStartup?: boolean;
272272
readonly continueThreadsAfterServerUpdate?: boolean;
273+
readonly eventSink?: (sink: EventSink.EventSinkV2Shape) => EventSink.EventSinkV2Shape;
273274
} = {},
274275
): Layer.Layer<
275276
| Orchestrator.OrchestratorV2
@@ -311,9 +312,19 @@ export function layerWithRegistry<Error>(
311312
EffectOutbox.layer,
312313
TurnItemPositionStore.layer,
313314
).pipe(Layer.provide(layerDatabase));
314-
const layerEventSinkProvided = EventSink.layerFromStores.pipe(
315+
const layerStoreEventSink = EventSink.layerFromStores.pipe(
315316
Layer.provide(Layer.mergeAll(layerStores, layerDatabase)),
316317
);
318+
const wrapEventSink = options.eventSink;
319+
const layerEventSinkProvided =
320+
wrapEventSink === undefined
321+
? layerStoreEventSink
322+
: Layer.effect(
323+
EventSink.EventSinkV2,
324+
Effect.gen(function* () {
325+
return wrapEventSink(yield* EventSink.EventSinkV2);
326+
}),
327+
).pipe(Layer.provide(layerStoreEventSink));
317328
const layerCommandReceiptStoreProvided = CommandReceiptStore.layer.pipe(
318329
Layer.provide(layerDatabase),
319330
);

0 commit comments

Comments
 (0)